From 86da6147f63b29cb85d51620b55bae7266f1c890 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Mon, 8 May 2017 16:07:43 +0200 Subject: [PATCH] Big backport from Enterprise --- bacula/.gitignore | 56 +- bacula/ChangeLog | 2 - bacula/LICENSE | 38 +- bacula/LICENSE-FAQ | 5 +- bacula/LICENSE-FOSS | 45 +- bacula/autoconf/config.guess | 324 ++-- bacula/autoconf/config.h.in | 36 + bacula/autoconf/config.sub | 68 +- bacula/autoconf/configure.in | 102 +- bacula/configure | 264 ++- bacula/platforms/redhat/bacula-dir.in | 10 +- bacula/platforms/redhat/bacula-sd.in | 10 +- bacula/platforms/systemd/bacula-sd.service.in | 3 +- bacula/release/bgit.py | 117 ++ bacula/scripts/Makefile.in | 11 +- bacula/scripts/tapealert | 68 + bacula/src/baconfig.h | 47 +- bacula/src/bacula.h | 31 +- bacula/src/bc_types.h | 6 +- bacula/src/c | 4 +- bacula/src/cats/bdb.h | 60 +- bacula/src/cats/bdb_mysql.h | 118 +- bacula/src/cats/bdb_postgresql.h | 5 +- bacula/src/cats/bdb_sqlite.h | 5 +- bacula/src/cats/bvfs.c | 227 ++- bacula/src/cats/bvfs.h | 93 +- bacula/src/cats/cats.h | 21 +- bacula/src/cats/create_postgresql_database.in | 2 +- bacula/src/cats/grant_mysql_privileges.in | 4 +- .../src/cats/grant_postgresql_privileges.in | 2 +- bacula/src/cats/make_bacula_tables.in | 2 +- bacula/src/cats/make_catalog_backup.in | 2 +- bacula/src/cats/make_catalog_backup.pl.in | 68 +- bacula/src/cats/make_mysql_tables.in | 38 +- bacula/src/cats/make_postgresql_tables.in | 16 +- bacula/src/cats/make_sqlite3_tables.in | 12 +- bacula/src/cats/mysql.c | 9 +- bacula/src/cats/postgresql.c | 4 +- bacula/src/cats/protos.h | 11 +- bacula/src/cats/sql.c | 199 ++- bacula/src/cats/sql_cmds.c | 24 +- bacula/src/cats/sql_cmds.h | 4 +- bacula/src/cats/sql_create.c | 82 +- bacula/src/cats/sql_delete.c | 7 +- bacula/src/cats/sql_find.c | 69 +- bacula/src/cats/sql_get.c | 220 ++- bacula/src/cats/sql_list.c | 135 +- bacula/src/cats/sql_update.c | 68 +- bacula/src/cats/update_bacula_tables.in | 2 +- bacula/src/cats/update_mysql_tables.in | 50 +- bacula/src/cats/update_postgresql_tables.in | 31 +- bacula/src/cats/update_sqlite3_tables.in | 50 +- bacula/src/ch.h | 6 +- bacula/src/console/Makefile.in | 11 +- bacula/src/console/authenticate.c | 25 +- bacula/src/console/bbconsjson.c | 594 +++++++ bacula/src/console/conio.c | 45 +- bacula/src/console/conio.h | 8 +- bacula/src/console/console.c | 221 ++- bacula/src/console/console_conf.c | 69 +- bacula/src/console/console_conf.h | 10 +- bacula/src/dird/Makefile.in | 17 +- bacula/src/dird/admin.c | 2 +- bacula/src/dird/authenticate.c | 28 +- bacula/src/dird/autoprune.c | 21 +- bacula/src/dird/backup.c | 120 +- bacula/src/dird/bacula-dir.conf.in | 13 +- bacula/src/dird/bdirjson.c | 1463 +++++++++++++++++ bacula/src/dird/bsr.c | 340 ++-- bacula/src/dird/bsr.h | 16 +- bacula/src/dird/catreq.c | 145 +- bacula/src/dird/dir_plugins.h | 6 +- bacula/src/dird/dird.c | 331 +++- bacula/src/dird/dird.h | 6 +- bacula/src/dird/dird_conf.c | 571 +++++-- bacula/src/dird/dird_conf.h | 104 +- bacula/src/dird/expand.c | 3 +- bacula/src/dird/fd_cmds.c | 36 +- bacula/src/dird/getmsg.c | 48 +- bacula/src/dird/job.c | 212 ++- bacula/src/dird/jobq.c | 93 +- bacula/src/dird/jobq.h | 22 +- bacula/src/dird/mac.c | 44 +- bacula/src/dird/msgchan.c | 16 +- bacula/src/dird/next_vol.c | 33 +- bacula/src/dird/protos.h | 40 +- bacula/src/dird/recycle.c | 8 +- bacula/src/dird/restore.c | 16 +- bacula/src/dird/scheduler.c | 8 +- bacula/src/dird/snapshot.c | 18 +- bacula/src/dird/ua.h | 18 +- bacula/src/dird/ua_acl.c | 28 +- bacula/src/dird/ua_cmds.c | 461 +++++- bacula/src/dird/ua_dotcmds.c | 632 ++++++- bacula/src/dird/ua_label.c | 82 +- bacula/src/dird/ua_output.c | 172 +- bacula/src/dird/ua_prune.c | 26 +- bacula/src/dird/ua_purge.c | 169 +- bacula/src/dird/ua_query.c | 2 +- bacula/src/dird/ua_restore.c | 119 +- bacula/src/dird/ua_run.c | 583 ++++++- bacula/src/dird/ua_select.c | 250 ++- bacula/src/dird/ua_status.c | 326 +++- bacula/src/dird/ua_tree.c | 18 +- bacula/src/dird/ua_update.c | 88 +- bacula/src/dird/vbackup.c | 104 +- bacula/src/dird/verify.c | 22 +- bacula/src/filed/Makefile.in | 13 +- bacula/src/filed/accurate.c | 92 +- bacula/src/filed/authenticate.c | 25 +- bacula/src/filed/backup.c | 12 +- bacula/src/filed/backup.h | 8 +- bacula/src/filed/bfdjson.c | 608 +++++++ bacula/src/filed/fd_plugins.h | 9 +- bacula/src/filed/fd_snapshot.c | 12 +- bacula/src/filed/filed.c | 92 +- bacula/src/filed/filed.h | 15 +- bacula/src/filed/filed_conf.c | 131 +- bacula/src/filed/filed_conf.h | 30 +- bacula/src/filed/heartbeat.c | 10 +- bacula/src/filed/hello.c | 172 +- bacula/src/filed/job.c | 307 +++- bacula/src/filed/protos.h | 7 +- bacula/src/filed/restore.c | 52 +- bacula/src/filed/restore.h | 5 +- bacula/src/filed/status.c | 139 +- bacula/src/filed/verify_vol.c | 83 +- bacula/src/filed/xattr.h | 76 + bacula/src/fileopts.h | 6 +- bacula/src/filetypes.h | 6 +- bacula/src/findlib/attribs.c | 20 +- bacula/src/findlib/bfile.c | 2 +- bacula/src/findlib/fstype.c | 2 +- bacula/src/findlib/protos.h | 4 +- bacula/src/host.h.in | 3 +- bacula/src/jcr.h | 45 +- bacula/src/lib/Makefile.in | 118 +- bacula/src/lib/alist.c | 286 +++- bacula/src/lib/alist.h | 86 +- bacula/src/lib/attr.c | 15 +- bacula/src/lib/bget_msg.c | 6 +- bacula/src/lib/bget_msg.h | 4 +- bacula/src/lib/bjson.c | 408 +++++ bacula/src/lib/bjson.h | 57 + bacula/src/lib/bnet.c | 66 +- bacula/src/lib/bnet_server.c | 2 +- bacula/src/lib/bpipe.c | 81 +- bacula/src/lib/bsock.c | 418 ++++- bacula/src/lib/bsock.h | 103 +- bacula/src/lib/bsys.c | 360 +++- bacula/src/lib/btimers.c | 9 +- bacula/src/lib/bwlimit.c | 64 + bacula/src/lib/bwlimit.h | 43 + bacula/src/lib/cmd_parser.h | 196 +++ bacula/src/lib/daemon.c | 14 +- bacula/src/lib/edit.c | 2 +- bacula/src/lib/flist.c | 152 ++ bacula/src/lib/flist.h | 95 ++ bacula/src/lib/htable.c | 8 +- bacula/src/lib/ini.c | 15 +- bacula/src/lib/jcr.c | 71 +- bacula/src/lib/lex.c | 27 +- bacula/src/lib/lex.h | 2 + bacula/src/lib/lib.h | 6 +- bacula/src/lib/lockmgr.c | 129 +- bacula/src/lib/lockmgr.h | 50 +- bacula/src/lib/lz4.c | 685 ++++++++ bacula/src/lib/lz4.h | 165 ++ bacula/src/lib/lz4_encoder.h | 258 +++ bacula/src/lib/mem_pool.c | 13 +- bacula/src/lib/mem_pool.h | 3 +- bacula/src/lib/message.c | 118 +- bacula/src/lib/message.h | 7 +- bacula/src/lib/output.c | 496 ++++++ bacula/src/lib/output.h | 159 ++ bacula/src/lib/parse_conf.c | 306 +++- bacula/src/lib/parse_conf.h | 96 +- bacula/src/lib/protos.h | 63 +- bacula/src/lib/rblist.c | 8 +- bacula/src/lib/res.c | 48 +- bacula/src/lib/scan.c | 18 +- bacula/src/lib/sha2.c | 950 +++++++++++ bacula/src/lib/sha2.h | 118 ++ bacula/src/lib/signal.c | 71 +- bacula/src/lib/smartall.c | 7 +- bacula/src/lib/smartall.h | 6 +- bacula/src/lib/status.h | 37 +- bacula/src/lib/tls.c | 46 +- bacula/src/lib/tree.c | 6 +- bacula/src/lib/tree.h | 3 +- bacula/src/lib/util.c | 49 +- bacula/src/lib/watchdog.c | 17 +- bacula/src/lib/watchdog.h | 18 +- bacula/src/lib/worker.c | 437 +++++ bacula/src/lib/worker.h | 103 ++ bacula/src/lib/workq.c | 164 +- bacula/src/lib/workq.h | 6 +- bacula/src/plugins/dir/example-plugin-dir.c | 6 +- bacula/src/plugins/fd/fd_common.h | 719 ++++++-- bacula/src/plugins/sd/example-plugin-sd.c | 6 +- bacula/src/plugins/sd/main.c | 5 +- bacula/src/qt-console/bat.h | 3 + bacula/src/qt-console/bat.pro.in | 2 + bacula/src/qt-console/bat_conf.cpp | 56 +- bacula/src/qt-console/bat_conf.h | 7 +- bacula/src/qt-console/bcomm/dircomm_auth.cpp | 26 +- bacula/src/qt-console/help/restore.html | 19 - .../src/qt-console/images/ajax-loader-big.gif | Bin 0 -> 3208 bytes bacula/src/qt-console/main.cpp | 2 +- bacula/src/qt-console/main.qrc | 1 + bacula/src/qt-console/medialist/mediaview.cpp | 6 +- .../qt-console/tray-monitor/authenticate.cpp | 197 +-- bacula/src/qt-console/tray-monitor/common.h | 67 + bacula/src/qt-console/tray-monitor/conf.cpp | 412 +++++ .../qt-console/tray-monitor/dir-monitor.ui | 176 ++ .../src/qt-console/tray-monitor/dirstatus.cpp | 127 ++ .../src/qt-console/tray-monitor/dirstatus.h | 42 + .../src/qt-console/tray-monitor/fd-monitor.ui | 176 ++ .../src/qt-console/tray-monitor/fdstatus.cpp | 125 ++ bacula/src/qt-console/tray-monitor/fdstatus.h | 42 + .../tray-monitor/install_conf_file.in | 16 + bacula/src/qt-console/tray-monitor/task.h | 115 ++ .../qt-console/tray-monitor/tray-monitor.cpp | 539 +++--- .../qt-console/tray-monitor/tray-monitor.h | 163 +- .../tray-monitor/tray-monitor.pro.in | 13 +- .../tray-monitor/tray-monitor.pro.mingw32.in | 12 +- .../tray-monitor/tray-monitor.pro.mingw64.in | 12 +- bacula/src/qt-console/tray-monitor/tray-ui.h | 402 ++--- .../src/qt-console/tray-monitor/tray_conf.cpp | 252 +-- .../src/qt-console/tray-monitor/tray_conf.h | 160 +- bacula/src/qt-console/util/fmtwidgetitem.cpp | 2 +- bacula/src/stored/Makefile.in | 227 ++- bacula/src/stored/acquire.c | 99 +- bacula/src/stored/aligned_dev.c | 21 + bacula/src/stored/aligned_dev.h | 143 ++ bacula/src/stored/aligned_read.c | 25 + bacula/src/stored/aligned_write.c | 26 + bacula/src/stored/ansi_label.c | 9 +- bacula/src/stored/append.c | 27 +- bacula/src/stored/askdir.c | 334 ++-- bacula/src/stored/authenticate.c | 20 +- bacula/src/stored/autochanger.c | 149 +- bacula/src/stored/bacula-sd.conf.in | 62 +- bacula/src/stored/bcopy.c | 49 +- bacula/src/stored/bextract.c | 76 +- bacula/src/stored/block.c | 429 +++-- bacula/src/stored/block.h | 48 +- bacula/src/stored/block_util.c | 356 +++- bacula/src/stored/bls.c | 120 +- bacula/src/stored/bscan.c | 112 +- bacula/src/stored/bsdjson.c | 680 ++++++++ bacula/src/stored/btape.c | 194 ++- bacula/src/stored/butil.c | 7 +- bacula/src/stored/cloud_dev.c | 18 + bacula/src/stored/cloud_dev.h | 78 + bacula/src/stored/cloud_driver.h | 57 + bacula/src/stored/cloud_parts.c | 18 + bacula/src/stored/cloud_parts.h | 23 + bacula/src/stored/cloud_test.c | 19 + bacula/src/stored/cloud_transfer_mgr.c | 26 + bacula/src/stored/cloud_transfer_mgr.h | 26 + bacula/src/stored/dev.c | 1059 ++++++------ bacula/src/stored/dev.h | 322 ++-- bacula/src/stored/device.c | 97 +- bacula/src/stored/dircmd.c | 466 +++++- bacula/src/stored/fd_cmds.c | 92 +- bacula/src/stored/fifo_dev.c | 51 + bacula/src/stored/fifo_dev.h | 39 + bacula/src/stored/file_dev.c | 306 +++- bacula/src/stored/file_dev.h | 9 +- bacula/src/stored/file_driver.c | 31 + bacula/src/stored/file_driver.h | 40 + bacula/src/stored/global.c | 26 + bacula/src/stored/hello.c | 38 +- bacula/src/stored/init_dev.c | 474 ++++++ bacula/src/stored/job.c | 5 +- bacula/src/stored/label.c | 576 ++++--- bacula/src/stored/lock.c | 96 +- bacula/src/stored/lock.h | 5 +- bacula/src/stored/match_bsr.c | 366 +++-- bacula/src/stored/mount.c | 167 +- bacula/src/stored/null_dev.c | 33 + bacula/src/stored/null_dev.h | 34 + bacula/src/stored/os.c | 4 +- bacula/src/stored/parse_bsr.c | 43 +- bacula/src/stored/protos.h | 100 +- bacula/src/stored/read.c | 21 +- bacula/src/stored/read_records.c | 251 ++- bacula/src/stored/record.h | 57 +- bacula/src/stored/record_read.c | 74 +- bacula/src/stored/record_util.c | 49 +- bacula/src/stored/record_write.c | 148 +- bacula/src/stored/reserve.c | 340 ++-- bacula/src/stored/reserve.h | 5 +- bacula/src/stored/s3_driver.c | 806 +++++++++ bacula/src/stored/s3_driver.h | 67 + bacula/src/stored/scan.c | 8 +- bacula/src/stored/sd_plugins.c | 34 +- bacula/src/stored/spool.c | 203 ++- bacula/src/stored/status.c | 457 ++++- bacula/src/stored/stored.c | 112 +- bacula/src/stored/stored.h | 12 +- bacula/src/stored/stored_conf.c | 358 +++- bacula/src/stored/stored_conf.h | 70 +- bacula/src/stored/tape_alert.c | 230 +++ bacula/src/stored/tape_alert_msgs.h | 166 ++ bacula/src/stored/tape_dev.c | 181 +- bacula/src/stored/tape_dev.h | 36 +- bacula/src/stored/vbackup.c | 24 +- bacula/src/stored/vol_mgr.c | 8 +- bacula/src/stored/vtape_dev.c | 38 +- bacula/src/stored/vtape_dev.h | 14 +- bacula/src/stored/wait.c | 60 +- bacula/src/streams.h | 9 +- bacula/src/tools/Makefile.in | 13 +- bacula/src/tools/cats_test.c | 2 +- bacula/src/tools/dbcheck.c | 2 +- bacula/src/tools/testfind.c | 5 +- bacula/src/version.h | 8 +- regress/all-changer-tests | 12 +- regress/all-disk-tests | 123 +- regress/all-tape-tests | 18 +- regress/do_all_tests | 18 +- regress/scripts/aligned-bacula-dir.conf.in | 822 +++++++++ regress/scripts/aligned-bacula-sd.conf.in | 143 ++ regress/scripts/ansi-sd-tape.conf.in | 2 + regress/scripts/bacula-dir-2client.conf.in | 4 + .../scripts/bacula-dir-2media-virtual.conf.in | 253 +++ regress/scripts/bacula-dir-tape.conf.in | 151 +- regress/scripts/bacula-dir-vtape.conf.in | 25 +- regress/scripts/bacula-dir-vtape2.conf.in | 423 +++++ .../scripts/bacula-dir.conf.testrunscript.in | 22 + regress/scripts/bacula-sd-2d.conf.in | 11 + regress/scripts/bacula-sd-2disk-drive.conf.in | 22 +- .../scripts/bacula-sd-2disk-virtual.conf.in | 253 +++ regress/scripts/bacula-sd-2disk.conf.in | 20 +- .../scripts/bacula-sd-2media-virtual.conf.in | 403 +++++ regress/scripts/bacula-sd-2tape.conf.in | 12 + regress/scripts/bacula-sd-4disk.conf.in | 184 +++ regress/scripts/bacula-sd-btape.conf.in | 12 + regress/scripts/bacula-sd-migration.conf.in | 18 +- regress/scripts/bacula-sd-tape.conf.in | 13 + .../scripts/bacula-sd-virtual-tape.conf.in | 80 + regress/scripts/bacula-sd-virtual.conf.in | 12 + regress/scripts/bacula-sd-vtape.conf.in | 12 + .../broken-media-bug-2-bacula-sd.conf.in | 11 + regress/scripts/cleanup | 46 +- regress/scripts/copy-2client-confs | 5 +- regress/scripts/copy-2disk-confs | 6 +- regress/scripts/copy-2disk-drive-confs | 5 +- regress/scripts/copy-2disk-tape-confs | 14 + regress/scripts/copy-2disk-virtual-confs | 14 + regress/scripts/copy-2drive-confs | 5 +- regress/scripts/copy-2tape-confs | 5 +- regress/scripts/copy-4disk-confs | 17 + regress/scripts/copy-btape-confs | 3 +- regress/scripts/copy-tls-crypto-confs | 14 + regress/scripts/do_sed | 20 +- regress/scripts/functions | 268 ++- regress/scripts/functions.pm | 289 +++- regress/scripts/migrate-bacula-dir.conf.in | 755 +++++++++ regress/scripts/migrate-bacula-sd.conf.in | 187 +++ regress/scripts/new-test-bacula-dir.conf.in | 67 + regress/scripts/prepare-fake-autochanger.in | 13 +- regress/scripts/prepare-other-loc | 4 +- regress/scripts/prepare-other-loc2 | 27 + regress/scripts/regress-config.in | 3 +- regress/scripts/setup | 91 +- regress/scripts/test-bacula-sd.conf.in | 23 + regress/scripts/tls-auth-bacula-sd.conf.in | 11 + regress/scripts/tls-bacula-sd.conf.in | 11 + regress/scripts/tls-crypto-bacula-fd.conf.in | 47 + .../virtualfull-extreme-bacula-dir.conf.in | 28 +- regress/tests/2drive-offline-test | 46 +- regress/tests/2media-virtual-test | 71 + regress/tests/accurate-test | 2 +- regress/tests/action-on-purge-test | 70 +- regress/tests/aligned-bug-8013-test | 131 ++ regress/tests/aligned-test | 147 ++ regress/tests/ansi-label-tape | 6 +- regress/tests/auto-label-jobmedia-test | 109 ++ regress/tests/auto-label-many-test | 90 + regress/tests/auto-label-test | 4 +- regress/tests/backup-to-null | 9 +- regress/tests/base-job-test | 6 +- regress/tests/bls-test | 54 + regress/tests/bpipe-test | 160 ++ regress/tests/bscan-tape | 4 - regress/tests/bsr-test | 62 + regress/tests/btape-fill-full-changer | 8 +- regress/tests/btape-fill-full-tape | 7 +- regress/tests/btape-test | 108 ++ regress/tests/btape-test-changer | 12 +- regress/tests/cancel-inactive-test | 68 + regress/tests/cancel-test | 12 +- regress/tests/console-dotcmd-test | 113 ++ regress/tests/copy-job-test | 6 - regress/tests/copy-swap-fail-test | 159 ++ regress/tests/copy-uncopied-test | 5 +- regress/tests/copy-volume-test | 3 +- regress/tests/crazy-smaller-vol-test | 106 ++ regress/tests/four-drive-test | 88 + regress/tests/four-jobs-tape | 4 - regress/tests/incremental-2media | 1 - regress/tests/incremental-changer | 2 + regress/tests/jobmedia-bug-test | 117 ++ regress/tests/maxbw-test | 9 +- regress/tests/maxbytes-test | 4 +- regress/tests/maxuseduration-test | 9 +- regress/tests/mcj-test | 78 + regress/tests/messages-saved-test | 59 + regress/tests/multi-storage-test | 2 +- regress/tests/next-pool-test | 200 +++ regress/tests/next-vol-bug-7302 | 119 ++ regress/tests/next-vol-test | 15 +- regress/tests/poll-interval-test | 79 + regress/tests/pool-attributes-test | 109 ++ regress/tests/priority-test | 347 ++++ regress/tests/prune-pool-test | 151 ++ regress/tests/relabel-tape | 1 - regress/tests/remote-console-test | 108 ++ regress/tests/restart-reschedule-test | 130 ++ regress/tests/restart-sd-test | 76 + regress/tests/restart2-base-job-test | 93 ++ regress/tests/restart2-job-test | 129 ++ regress/tests/restore-multi-session-test | 138 ++ regress/tests/restore-stop-read-test | 145 ++ regress/tests/restore-stop-read2-test | 83 + regress/tests/scratch-pool-test | 4 +- regress/tests/scratchpool-pool-test | 4 +- regress/tests/sd-sd-test | 136 ++ regress/tests/source-addr-test | 10 +- regress/tests/three-pool-virtual-test | 89 + regress/tests/truncate-bug-tape | 2 +- regress/tests/truncate-test | 221 +++ regress/tests/truncate2-test | 229 +++ regress/tests/two-jobs-test | 2 +- regress/tests/two-pool-changer | 2 - regress/tests/virtual-jobid-test | 215 +++ regress/tests/virtualfull-bug-7154 | 85 + regress/tests/vtape-round-robin-changer | 97 ++ 441 files changed, 38188 insertions(+), 7688 deletions(-) create mode 100755 bacula/release/bgit.py create mode 100755 bacula/scripts/tapealert create mode 100644 bacula/src/console/bbconsjson.c create mode 100644 bacula/src/dird/bdirjson.c create mode 100644 bacula/src/filed/bfdjson.c create mode 100644 bacula/src/filed/xattr.h create mode 100644 bacula/src/lib/bjson.c create mode 100644 bacula/src/lib/bjson.h create mode 100644 bacula/src/lib/bwlimit.c create mode 100644 bacula/src/lib/bwlimit.h create mode 100644 bacula/src/lib/cmd_parser.h create mode 100644 bacula/src/lib/flist.c create mode 100644 bacula/src/lib/flist.h create mode 100644 bacula/src/lib/lz4.c create mode 100644 bacula/src/lib/lz4.h create mode 100644 bacula/src/lib/lz4_encoder.h create mode 100644 bacula/src/lib/output.c create mode 100644 bacula/src/lib/output.h create mode 100644 bacula/src/lib/sha2.c create mode 100644 bacula/src/lib/sha2.h create mode 100644 bacula/src/lib/worker.c create mode 100644 bacula/src/lib/worker.h create mode 100644 bacula/src/qt-console/images/ajax-loader-big.gif create mode 100644 bacula/src/qt-console/tray-monitor/common.h create mode 100644 bacula/src/qt-console/tray-monitor/conf.cpp create mode 100644 bacula/src/qt-console/tray-monitor/dir-monitor.ui create mode 100644 bacula/src/qt-console/tray-monitor/dirstatus.cpp create mode 100644 bacula/src/qt-console/tray-monitor/dirstatus.h create mode 100644 bacula/src/qt-console/tray-monitor/fd-monitor.ui create mode 100644 bacula/src/qt-console/tray-monitor/fdstatus.cpp create mode 100644 bacula/src/qt-console/tray-monitor/fdstatus.h create mode 100755 bacula/src/qt-console/tray-monitor/install_conf_file.in create mode 100644 bacula/src/qt-console/tray-monitor/task.h create mode 100644 bacula/src/stored/aligned_dev.c create mode 100644 bacula/src/stored/aligned_dev.h create mode 100644 bacula/src/stored/aligned_read.c create mode 100644 bacula/src/stored/aligned_write.c create mode 100644 bacula/src/stored/bsdjson.c create mode 100644 bacula/src/stored/cloud_dev.c create mode 100644 bacula/src/stored/cloud_dev.h create mode 100644 bacula/src/stored/cloud_driver.h create mode 100644 bacula/src/stored/cloud_parts.c create mode 100644 bacula/src/stored/cloud_parts.h create mode 100644 bacula/src/stored/cloud_test.c create mode 100644 bacula/src/stored/cloud_transfer_mgr.c create mode 100644 bacula/src/stored/cloud_transfer_mgr.h create mode 100644 bacula/src/stored/fifo_dev.c create mode 100644 bacula/src/stored/fifo_dev.h create mode 100644 bacula/src/stored/file_driver.c create mode 100644 bacula/src/stored/file_driver.h create mode 100644 bacula/src/stored/global.c create mode 100644 bacula/src/stored/init_dev.c create mode 100644 bacula/src/stored/null_dev.c create mode 100644 bacula/src/stored/null_dev.h create mode 100644 bacula/src/stored/s3_driver.c create mode 100644 bacula/src/stored/s3_driver.h create mode 100644 bacula/src/stored/tape_alert.c create mode 100644 bacula/src/stored/tape_alert_msgs.h create mode 100644 regress/scripts/aligned-bacula-dir.conf.in create mode 100644 regress/scripts/aligned-bacula-sd.conf.in create mode 100644 regress/scripts/bacula-dir-2media-virtual.conf.in create mode 100644 regress/scripts/bacula-dir-vtape2.conf.in create mode 100644 regress/scripts/bacula-sd-2disk-virtual.conf.in create mode 100644 regress/scripts/bacula-sd-2media-virtual.conf.in create mode 100644 regress/scripts/bacula-sd-4disk.conf.in create mode 100644 regress/scripts/bacula-sd-virtual-tape.conf.in create mode 100755 regress/scripts/copy-2disk-tape-confs create mode 100755 regress/scripts/copy-2disk-virtual-confs create mode 100755 regress/scripts/copy-4disk-confs create mode 100755 regress/scripts/copy-tls-crypto-confs create mode 100644 regress/scripts/migrate-bacula-dir.conf.in create mode 100644 regress/scripts/migrate-bacula-sd.conf.in create mode 100755 regress/scripts/prepare-other-loc2 create mode 100644 regress/scripts/tls-crypto-bacula-fd.conf.in create mode 100755 regress/tests/2media-virtual-test create mode 100755 regress/tests/aligned-bug-8013-test create mode 100755 regress/tests/aligned-test create mode 100755 regress/tests/auto-label-jobmedia-test create mode 100755 regress/tests/auto-label-many-test create mode 100755 regress/tests/bls-test create mode 100755 regress/tests/bpipe-test create mode 100755 regress/tests/bsr-test create mode 100755 regress/tests/btape-test create mode 100755 regress/tests/cancel-inactive-test create mode 100755 regress/tests/console-dotcmd-test create mode 100755 regress/tests/copy-swap-fail-test create mode 100755 regress/tests/crazy-smaller-vol-test create mode 100755 regress/tests/four-drive-test create mode 100755 regress/tests/jobmedia-bug-test create mode 100755 regress/tests/mcj-test create mode 100755 regress/tests/messages-saved-test create mode 100755 regress/tests/next-pool-test create mode 100755 regress/tests/next-vol-bug-7302 create mode 100755 regress/tests/poll-interval-test create mode 100755 regress/tests/pool-attributes-test create mode 100755 regress/tests/priority-test create mode 100755 regress/tests/prune-pool-test create mode 100755 regress/tests/remote-console-test create mode 100755 regress/tests/restart-reschedule-test create mode 100755 regress/tests/restart-sd-test create mode 100755 regress/tests/restart2-base-job-test create mode 100755 regress/tests/restart2-job-test create mode 100755 regress/tests/restore-multi-session-test create mode 100755 regress/tests/restore-stop-read-test create mode 100755 regress/tests/restore-stop-read2-test create mode 100755 regress/tests/sd-sd-test create mode 100755 regress/tests/three-pool-virtual-test create mode 100755 regress/tests/truncate-test create mode 100755 regress/tests/truncate2-test create mode 100755 regress/tests/virtual-jobid-test create mode 100755 regress/tests/virtualfull-bug-7154 create mode 100755 regress/tests/vtape-round-robin-changer diff --git a/bacula/.gitignore b/bacula/.gitignore index f909c3a487..92f3c4bf6e 100644 --- a/bacula/.gitignore +++ b/bacula/.gitignore @@ -13,12 +13,9 @@ *.lo *~ *.patch +*.bak 1 2 -*.py -*.c -*.sql -clientproductionconfig autom4te.cache bacula btraceback @@ -232,8 +229,6 @@ scripts/logwatch/logfile.bacula.conf src/1 src/2 src/3 -src/add-bsd -src/bsd src/Makefile src/config.h src/testprogs @@ -243,6 +238,8 @@ src/python src/pyqt src/xqt-console src/TAGS +src/bsd +src/add-bsd # src/cats/ src/cats/1 @@ -358,6 +355,21 @@ src/findlib/libfind.a src/findlib/.libs src/findlib/libbacfind.a +# src/gnome2-console/ +src/gnome2-console/test-gnome-console.conf +src/gnome2-console/Makefile +src/gnome2-console/gnome-console +src/gnome2-console/gnome-console.conf +src/gnome2-console/1 +src/gnome2-console/2 +src/gnome2-console/3 +src/gnome2-console/main.c +src/gnome2-console/static-gnome-console +src/gnome2-console/bgnome-console +src/gnome2-console/bgnome-console.conf +src/gnome2-console/support.c.orig +src/gnome2-console/.libs + # src/lib/ src/lib/bsnprintf src/lib/save @@ -396,15 +408,6 @@ src/plugins/sd/main src/plugins/sd/Makefile # src/qt-console/ -src/qt-console/tray-monitor/.libs/ -src/qt-console/tray-monitor/install_conf_file -src/qt-console/tray-monitor/object_script.bacula-tray-monitor.Debug -src/qt-console/tray-monitor/object_script.bacula-tray-monitor.Release -src/qt-console/tray-monitor/bacula-tray-monitor -src/qt-console/tray-monitor/bacula-tray-monitor.conf -src/qt-console/tray-monitor/bacula-tray-monitor.Debug -src/qt-console/tray-monitor/bacula-tray-monitor.Release -src/qt-console/tray-monitor/qrc_main.cpp src/qt-console/ui_*.h src/qt-console/bat src/qt-console/about-func @@ -490,6 +493,9 @@ src/qt-console/testprogs/putz/Makefile src/qt-console/testprogs/putz/moc # src/qt-console/tray-monitor +src/qt-console/tray-monitor/.libs/ +src/qt-console/tray-monitor/bacula-tray-monitor +src/qt-console/tray-monitor/qrc_main.cpp src/qt-console/tray-monitor/Makefile.mingw32 src/qt-console/tray-monitor/Makefile.mingw32.Debug src/qt-console/tray-monitor/Makefile.mingw32.Release @@ -508,6 +514,11 @@ src/qt-console/tray-monitor/moc32/ src/qt-console/tray-monitor/moc64/ src/qt-console/tray-monitor/tray-monitor.pro.mingw32 src/qt-console/tray-monitor/tray-monitor.pro.mingw64 +src/qt-console/tray-monitor/bacula-tray-monitor.conf +src/qt-console/tray-monitor/install_conf_file +src/qt-console/tray-monitor/object_script.bacula-tray-monitor.Debug +src/qt-console/tray-monitor/object_script.bacula-tray-monitor.Release + # src/qt-console/ts/ src/qt-console/ts/bat_fr.qm @@ -580,7 +591,6 @@ src/tray-monitor/*~ src/tray-monitor/.libs # src/win32/ -src/win32/*.log src/win32/Makefile.inc src/win32/release src/win32/pthreadGCE.dll @@ -808,8 +818,6 @@ src/wx-console/bwx-console.conf src/wx-console/.libs # updatedb/ -updatedb/update_mysql_tables_1018_to_1019 -updatedb/update_postgresql_tables_1018_to_1019 updatedb/update_mysql_tables updatedb/update_postgresql_tables updatedb/update_sqlite3_tables @@ -833,15 +841,17 @@ updatedb/update_mysql_tables_13_to_14 updatedb/update_mysql_tables_14_to_1014 updatedb/update_mysql_tables_1015_to_1016 updatedb/update_mysql_tables_1016_to_1017 -updatedb/update_mysql_tables_1017_to_1018 updatedb/update_postgresql_tables_1014_to_1015 updatedb/update_postgresql_tables_12_to_13 updatedb/update_postgresql_tables_13_to_14 updatedb/update_postgresql_tables_14_to_1014 updatedb/update_postgresql_tables_1015_to_1016 updatedb/update_postgresql_tables_1016_to_1017 -updatedb/update_postgresql_tables_1017_to_1018 updatedb/update_sqlite3_tables_1015_to_1016 +updatedb/update_mysql_tables_1017_to_1018 +updatedb/update_mysql_tables_1018_to_1019 +updatedb/update_postgresql_tables_1017_to_1018 +updatedb/update_postgresql_tables_1018_to_1019 # /docs/ @@ -1184,12 +1194,6 @@ updatedb/update_sqlite3_tables_1015_to_1016 /gui/bimagemgr/bacula-bimagemgr.spec # /regress/ -../regress/backup-mysql-7.0.sql -../regress/backup-postgres-5.2.sql -../regress/bacula-postgres-7.0.sql -../regress/bin-5.2.0/ -../regress/bin-7.0/ -/regress/do_database_ugrade /regress/testtime.out /regress/kerns.config /regress/1 diff --git a/bacula/ChangeLog b/bacula/ChangeLog index 568e662ae0..e8f45cca82 100644 --- a/bacula/ChangeLog +++ b/bacula/ChangeLog @@ -3035,7 +3035,6 @@ Version 5.2.0rc1 mdb as its rather rudimentary and more a crash debugger then a source code debugger. - Fix for bug #1560 bcopy cannot find Volume -- Add beef - Update autoconf scripts - Fix Windows build @@ -4215,7 +4214,6 @@ Release Version 4.0.0-RC1 24Apr10 - Fix for bug #1560 bcopy cannot find Volume -- Add BEEF in version.h 23Apr10 - Add compress/decompress of Object Record data diff --git a/bacula/LICENSE b/bacula/LICENSE index 10d15ca96c..b6cb222218 100644 --- a/bacula/LICENSE +++ b/bacula/LICENSE @@ -1,12 +1,11 @@ - Last major revision: 06 May 2014 - Last minor revision: 16 July 2016 + Last revision: 03 April 2017 Bacula is licensed under the GNU Affero General Public License, version 3.0 as published by the Free Software Foundation, Inc. ("AGPLv3"). -Additional Terms on the work licensed herein, pursuant to Section 7 of Affero -General Public License are as follows: +Additional Terms on the work licensed herein, pursuant to Section 7 of +Affero General Public License are as follows: 1. The name Bacula is a registered trademark of Kern Sibbald, and Kern Sibbald hereby declines to grant a trademark license to "Bacula" @@ -27,7 +26,7 @@ General Public License are as follows: Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -37,8 +36,8 @@ General Public License are as follows: Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is - conveyed and/or propagated. + This notice must be preserved when any source code is conveyed + and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -62,28 +61,29 @@ AGPLv3 are as follows: The copyright for certain source files may include in addition to what is listed above the following copyright: - Copyright (C) 2000-2016 Free Software Foundation Europe e.V. + Copyright (C) 2000-2014 Free Software Foundation Europe e.V. The copyright on the Baculum code is: - Copyright (C) 2013-2016 Marcin Haba + Copyright (C) 2013-2017 Marcin Haba -Copyrights of certain "script" files such as headers, shell script, -Makefiles, etc ... were previously not explicitly defined. In almost all -cases, they are now copyrighted with a BSD 2-Clause copyright to make them +Copyrights of certain "script" files such as headers, shell script, Makefiles, +etc ... were previously never explicitly defined. In almost all cases, +they have been copyrighted with a BSD 2-Clause copyright to make them easier. However, as is the case of all BSD type copyrights you must keep -the copyright in place and on any binary only released, the copyright notice -must also be released with the binaries. An example of such a copyright is: +the copyright in place and on any binary only released the copyright notice +must also be released with the binaries. An example of such a copyright +is: # -# Copyright (C) 2000-2016 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # It is equivalent to the full BSD copyright of: ===== -Copyright (C) 2000-2016 Kern Sibbald +Copyright (C) 2000-2017 Kern Sibbald License: BSD 2-Clause; see file LICENSE-FOSS Redistribution and use in source and binary forms, with or without @@ -108,9 +108,9 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==== -Note: the exact form of the copyright (dates, name) might vary, but -the intent is the same, namely that the full BSD 2-Clause copyright -applies. The file LICENSE-FOSS has a few more details. +Note: the exact form of the copyright (dates, name, and text formatting) +might vary, but the intent is the same, namely that the full BSD 2-Clause +coypright applies. The file LICENSE-FOSS has a few more details. ###################################################################### diff --git a/bacula/LICENSE-FAQ b/bacula/LICENSE-FAQ index a5e25839c6..d108a6b95d 100644 --- a/bacula/LICENSE-FAQ +++ b/bacula/LICENSE-FAQ @@ -38,7 +38,7 @@ License: To the best of our knowledge, all code used in Bacula, which is copyrighted by a third party, has licenses that are compatible with the OpenSSL license, and so given the exceptions that we have -made to the AGPLv3 above, Bacula can be freely linked and distributed +made to the AGPLv3 (in LICENSE), Bacula can be freely linked and distributed with the OpenSSL libraries, and in binary form with the Microsoft VSS libraries. @@ -62,7 +62,8 @@ Each Contributor to Bacula represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. -Other Licenses used in some source files: +Other Licenses used in some source files. Note, this list changes from +time to time, so is not exhaustive: GPLv2 or later license: src/tools/bsmtp.c diff --git a/bacula/LICENSE-FOSS b/bacula/LICENSE-FOSS index 33d32b36e2..e34e3c95ea 100644 --- a/bacula/LICENSE-FOSS +++ b/bacula/LICENSE-FOSS @@ -45,23 +45,24 @@ sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in the LICENSE file.. Notices: -The Bacula community version uses a certain number of files that have FOSS -(Free or Open Software) licenses. Most 3rd party FOSS licenses require no -notification so they are not included here. - -However, the Bacula Community binary releases consist of files, with few -source files (mostly scripts), and some of the FOSS licenses such such as -the BSD (Berkeley Software Development) require publication of the -copyright notices if the code is released in binary format. For example -certain copyright notifications are simplified such as: - -# Copyright (C) 2000-2016 Kern Sibbald +The Bacula community version uses a certain number of files that have +FOSS (Free or Open Software) licenses. Many of these files come from and are +identical to the Bacula community code. Most 3rd party FOSS licenses require +no notification so they are not included here. + +However, the Bacula Community binary releases consist of +files, with few source files (mostly scripts), and some of the FOSS licenses +such such as the BSD (Berkeley Software Development) require publication of +the copyright notices if the code is released in binary format. For example +certain copyright notifications are simplifed such as: + +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS This license corresponds to the following: ===== -Copyright (C) 2000-2016 Kern Sibbald +Copyright (C) 2000-2017 Kern Sibbald License: BSD 2-Clause; see file LICENSE-FOSS Redistribution and use in source and binary forms, with or without @@ -86,19 +87,19 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==== -The release of binaries mentioned above is covered in point 2 (above) of -license. This file contains a list of all such licenses. +The release of binaires mentioned above is covered in point 2 (above) of +license. This file contains a non-exhaustive list of such licenses. Database scripts (src/cats): -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS Translations (po): -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # Copyright (C) 2010 Inteos Sp. z o.o. -# Copyright (C) 2010-2014 Kern Sibbald +# Copyright (C) 2010-2017 Kern Sibbald # License: BSD 2-Clause # Copyright (C) 2005-2006 Free Software Foundation Europe e.V. @@ -107,16 +108,16 @@ Translations (po): Manpages: This man page document is released under the BSD 2-Clause license. -Update database (updatedb): -# Copyright (C) 2000-2016 Kern Sibbald +Update datbase (updatedb): +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS Scripts (scripts): -# Copyright (C) 2000-2016 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS Platforms (platforms): -# Copyright (C) 2000-2016 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS Libraries (src/lib): @@ -279,7 +280,7 @@ lz4.c lz4_encoder.h lz4.h You can contact the author at : - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/lz4/lz4 */ === diff --git a/bacula/autoconf/config.guess b/bacula/autoconf/config.guess index 981c376037..373a659a06 100755 --- a/bacula/autoconf/config.guess +++ b/bacula/autoconf/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2013 Free Software Foundation, Inc. +# Copyright 1992-2016 Free Software Foundation, Inc. -timestamp='2013-06-10' +timestamp='2016-02-11' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -24,12 +24,12 @@ timestamp='2013-06-10' # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # -# Originally written by Per Bothner. +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to . me=`echo "$0" | sed -e 's,.*/,,'` @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2013 Free Software Foundation, Inc. +Copyright 1992-2016 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -149,7 +149,7 @@ Linux|GNU|GNU/*) LIBC=gnu #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` ;; esac @@ -168,20 +168,27 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + /sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || \ + echo unknown)` case "${UNAME_MACHINE_ARCH}" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; + earmv*) + arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` + machine=${arch}${endian}-unknown + ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched # to ELF recently, or will in the future. case "${UNAME_MACHINE_ARCH}" in - arm*|i386|m68k|ns32k|sh3*|sparc|vax) + arm*|earm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ @@ -197,6 +204,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "${UNAME_MACHINE_ARCH}" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need @@ -207,13 +221,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "${machine}-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` @@ -223,6 +237,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} exit ;; @@ -235,6 +253,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} exit ;; + *:Sortix:*:*) + echo ${UNAME_MACHINE}-unknown-sortix + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -251,42 +272,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -359,16 +380,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval $set_cc_for_build - SUN_ARCH="i386" + SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then - SUN_ARCH="x86_64" + SUN_ARCH=x86_64 fi fi echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` @@ -393,7 +414,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) echo m68k-sun-sunos${UNAME_RELEASE} @@ -579,8 +600,9 @@ EOF else IBM_ARCH=powerpc fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi @@ -617,13 +639,13 @@ EOF sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi @@ -666,7 +688,7 @@ EOF test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ ${HP_ARCH} = hppa2.0w ] then eval $set_cc_for_build @@ -682,9 +704,9 @@ EOF if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} @@ -789,14 +811,14 @@ EOF echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) @@ -826,7 +848,7 @@ EOF *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 exit ;; - i*:MSYS*:*) + *:MSYS*:*) echo ${UNAME_MACHINE}-pc-msys exit ;; i*:windows32*:*) @@ -901,7 +923,7 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; arc:Linux:*:* | arceb:Linux:*:*) @@ -932,6 +954,9 @@ EOF crisv32:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; + e2k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; frv:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -944,6 +969,9 @@ EOF ia64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; + k1om:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -969,10 +997,10 @@ EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; - or1k:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + openrisc*:Linux:*:*) + echo or1k-unknown-linux-${LIBC} exit ;; - or32:Linux:*:*) + or32:Linux:*:* | or1k*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; padre:Linux:*:*) @@ -1001,9 +1029,6 @@ EOF ppcle:Linux:*:*) echo powerpcle-unknown-linux-${LIBC} exit ;; - ppc64el:Linux:*:*) - echo powerpc64le-unknown-linux-${LIBC} - exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo ${UNAME_MACHINE}-ibm-linux-${LIBC} exit ;; @@ -1023,7 +1048,7 @@ EOF echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; xtensa*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} @@ -1102,7 +1127,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1263,22 +1288,32 @@ EOF if test "$UNAME_PROCESSOR" = unknown ; then UNAME_PROCESSOR=powerpc fi - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac + if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # Avoid executing cc on OS X 10.9, as it ships with a stub + # that puts up a graphical alert prompting to install + # developer tools. Any system running Mac OS X 10.7 or + # later (Darwin 11 and later) is required to have a 64-bit + # processor. This is not true of the ARM version of Darwin + # that Apple uses in portable devices. + UNAME_PROCESSOR=x86_64 fi echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi @@ -1309,7 +1344,7 @@ EOF # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" @@ -1362,155 +1397,10 @@ EOF x86_64:VMkernel:*:*) echo ${UNAME_MACHINE}-unknown-esx exit ;; -esac - -eval $set_cc_for_build -cat >$dummy.c < -# include -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix\n"); exit (0); -#endif - -#if defined (hp300) && !defined (hpux) - printf ("m68k-hp-bsd\n"); exit (0); -#endif - -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); - -#endif - -#if defined (vax) -# if !defined (ultrix) -# include -# if defined (BSD) -# if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -# else -# if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# endif -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# else - printf ("vax-dec-ultrix\n"); exit (0); -# endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} -EOF - -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. - -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } - -# Convex versions that predate uname can use getsysinfo(1) - -if [ -x /usr/convex/getsysinfo ] -then - case `getsysinfo -f cpu_type` in - c1*) - echo c1-convex-bsd - exit ;; - c2*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - c34*) - echo c34-convex-bsd - exit ;; - c38*) - echo c38-convex-bsd - exit ;; - c4*) - echo c4-convex-bsd + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs exit ;; - esac -fi +esac cat >&2 < and . */ #undef TIME_WITH_SYS_TIME diff --git a/bacula/autoconf/config.sub b/bacula/autoconf/config.sub index 9633db7046..6223dde931 100755 --- a/bacula/autoconf/config.sub +++ b/bacula/autoconf/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2013 Free Software Foundation, Inc. +# Copyright 1992-2016 Free Software Foundation, Inc. -timestamp='2013-08-10' +timestamp='2016-01-01' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -25,7 +25,7 @@ timestamp='2013-08-10' # of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to . # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -33,7 +33,7 @@ timestamp='2013-08-10' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -53,8 +53,7 @@ timestamp='2013-08-10' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. @@ -68,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2013 Free Software Foundation, Inc. +Copyright 1992-2016 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -117,7 +116,7 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ kopensolaris*-gnu* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os @@ -255,16 +254,18 @@ case $basic_machine in | arc | arceb \ | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ | avr | avr32 \ + | ba \ | be32 | be64 \ | bfin \ | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i860 | i960 | ia64 \ | ip2k | iq2000 \ + | k1om \ | le32 | le64 \ | lm32 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ @@ -282,8 +283,10 @@ case $basic_machine in | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ | mipsr5900 | mipsr5900el \ @@ -295,14 +298,14 @@ case $basic_machine in | nds32 | nds32le | nds32be \ | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ - | open8 \ - | or1k | or32 \ + | open8 | or1k | or1knd | or32 \ | pdp10 | pdp11 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ + | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -310,6 +313,7 @@ case $basic_machine in | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | visium \ | we32k \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) @@ -324,7 +328,10 @@ case $basic_machine in c6x) basic_machine=tic6x-unknown ;; - m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; + m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) basic_machine=$basic_machine-unknown os=-none ;; @@ -369,18 +376,20 @@ case $basic_machine in | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ + | e2k-* | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ | i*86-* | i860-* | i960-* | ia64-* \ | ip2k-* | iq2000-* \ + | k1om-* \ | le32-* | le64-* \ | lm32-* \ | m32c-* | m32r-* | m32rle-* \ @@ -400,8 +409,10 @@ case $basic_machine in | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ | mipsr5900-* | mipsr5900el-* \ @@ -413,16 +424,18 @@ case $basic_machine in | nios-* | nios2-* | nios2eb-* | nios2el-* \ | none-* | np1-* | ns16k-* | ns32k-* \ | open8-* \ + | or1k*-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pyramid-* \ + | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ | tile*-* \ @@ -430,6 +443,7 @@ case $basic_machine in | ubicom32-* \ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ + | visium-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ @@ -506,6 +520,9 @@ case $basic_machine in basic_machine=i386-pc os=-aros ;; + asmjs) + basic_machine=asmjs-unknown + ;; aux) basic_machine=m68k-apple os=-aux @@ -767,6 +784,9 @@ case $basic_machine in basic_machine=m68k-isi os=-sysv ;; + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'` + ;; m68knommu) basic_machine=m68k-unknown os=-linux @@ -822,6 +842,10 @@ case $basic_machine in basic_machine=powerpc-unknown os=-morphos ;; + moxiebox) + basic_machine=moxie-unknown + os=-moxiebox + ;; msdos) basic_machine=i386-pc os=-msdos @@ -1354,11 +1378,11 @@ case $os in | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ + | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ + | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ @@ -1367,14 +1391,15 @@ case $os in | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* \ + | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ + | -onefs* | -tirtos*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1592,9 +1617,6 @@ case $basic_machine in mips*-*) os=-elf ;; - or1k-*) - os=-elf - ;; or32-*) os=-coff ;; diff --git a/bacula/autoconf/configure.in b/bacula/autoconf/configure.in index 7eed901272..f78bedc8e7 100644 --- a/bacula/autoconf/configure.in +++ b/bacula/autoconf/configure.in @@ -2,7 +2,7 @@ dnl dnl dnl Process this file with autoconf to produce a configure script. dnl -dnl Copyright (C) 2000-2016 Kern Sibbald +dnl Copyright (C) 2000-2017 Kern Sibbald dnl License: BSD 2-Clause; see file LICENSE-FOSS dnl dnl require a recent autoconf @@ -38,6 +38,10 @@ BDB_VERSION=`sed -n -e 's/^#define BDB_VERSION \(.*\)$/\1/p' ${srcdir}/src/cats/ DEPKGS_VERSION=`sed -n -e 's/^#define DEPKGS_VERSION \(.*\)$/\1/p' ${srcdir}/src/cats/cats.h` DEPKGS_QT_VERSION=`sed -n -e 's/^#define DEPKGS_QT_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` BQT_VERSION=`sed -n -e 's/^#define BQT_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +VIX_VERSION=`sed -n -e 's/^#define VIX_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +JAVA_VERSION=`sed -n -e 's/^#define JAVA_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +NDMP_VERSION=`sed -n -e 's/^#define NDMP_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +LIBRSYNC_VERSION=`sed -n -e 's/^#define LIBRSYNC_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` AC_SUBST(VERSION)dnl AC_SUBST(DATE)dnl AC_SUBST(LSMDATE)dnl @@ -47,6 +51,11 @@ AC_SUBST(BDB_VERSION)dnl AC_SUBST(DEPKGS_QT_VERSION)dnl AC_SUBST(DEPKGS_VERSION)dnl AC_SUBST(BQT4_VERSION)dnl +AC_SUBST(VIX_VERSION)dnl +AC_SUBST(JAVA_VERSION)dnl +AC_SUBST(NDMP_VERSION)dnl +AC_SUBST(LIBRSYNC_VERSION)dnl + dnl src/lib dnl can be overwritten by specific values from version.h @@ -79,7 +88,6 @@ LIBBACFIND_LT_RELEASE=${LIBBACFIND_LT_RELEASE:-$VERSION} AC_SUBST(LIBBACFIND_LT_RELEASE)dnl - dnl PFILES are platform or plugin specific files PFILES="platforms/Makefile" @@ -848,7 +856,7 @@ AC_SUBST(MAKE_SHELL) AC_HEADER_STAT AC_HEADER_DIRENT AC_CHECK_FUNCS(strcasecmp select setenv putenv tcgetattr) -AC_CHECK_FUNCS(lstat lchown lchmod futimes fchmod fchown) +AC_CHECK_FUNCS(lstat lchown lchmod futimes fchmod fchown lutimes) AC_CHECK_FUNCS(nanosleep nl_langinfo) AC_CHECK_FUNCS(be64toh htobe64) AC_CHECK_HEADERS(varargs.h) @@ -1887,6 +1895,93 @@ AC_MSG_RESULT($fstype) AC_CHECK_HEADER(sys/statvfs.h, [ AC_DEFINE(HAVE_SYS_STATVFS_H,1,[Defines if your system have the sys/statvfs.h header file])] , ) +AC_CHECK_DECLS([O_CLOEXEC],,[AC_DEFINE([O_CLOEXEC],[0], [Defined to 0 if not provided])], +[[ +#ifdef HAVE_FCNTL_H +# include +#endif +]]) + +AC_CHECK_DECLS([FD_CLOEXEC],,[AC_DEFINE([FD_CLOEXEC],[0], [Defined to 0 if not provided])], +[[ +#ifdef HAVE_FCNTL_H +# include +#endif +]]) + +AC_CHECK_DECLS([SOCK_CLOEXEC],,[AC_DEFINE([SOCK_CLOEXEC],[0],[Defined to 0 if not provided])], +[[ +#ifdef HAVE_SYS_SOCKET_H +# include +#endif +]]) + +AC_CACHE_CHECK(for close on exec modifier for fopen(), ac_cv_feature_stream_cloexec_flag, + [if test $ac_cv_have_decl_O_CLOEXEC = yes ; then + if test $ac_cv_have_decl_SOCK_CLOEXEC = yes ; then + ac_cv_feature_stream_cloexec_flag="e" + fi + fi]) + +if test "x$ac_cv_feature_stream_cloexec_flag" = "xe" ; then + AC_DEFINE(HAVE_STREAM_CLOEXEC,[0],[Defined to 0 if not provided]) +fi + +AC_DEFINE_UNQUOTED([STREAM_CLOEXEC], "$ac_cv_feature_stream_cloexec_flag", [fopen() modifier for setting close on exec flag]) + +AC_CHECK_FUNC(accept4, [AC_DEFINE(HAVE_ACCEPT4, 1, [Define to 1 if you have the 'accept4' function.])]) + +S3_INC= +S3_LIBS= +S3_LDFLAGS= +have_libs3=no + +if test x$support_s3 = xyes; then + AC_ARG_WITH(s3, + AC_HELP_STRING([--with-s3@<:@=DIR@:>@], [specify s3 library directory]), + [ + case "$with_s3" in + no) + : + ;; + yes|*) + if test -f ${with_s3}/include/libs3.h; then + S3_INC="-I${with_s3}/include" + S3_LDFLAGS="-L${with_s3}/lib" + with_s3="${with_s3}/include" + else + with_s3="/usr/include" + fi + + AC_CHECK_HEADER(${with_s3}/libs3.h, + [ + AC_DEFINE(HAVE_LIBS3, 1, [Define to 1 if you have libs3]) + S3_LIBS="${S3_LDFLAGS} -ls3" + have_libs3="yes" + ], [ + echo " " + echo "libs3.h not found. s3 turned off ..." + echo " " + ] + ) + ;; + esac + ],[ + AC_CHECK_HEADER(libs3.h, + [ + AC_CHECK_LIB(s3, S3_initialize, + [ + S3_LIBS="-ls3" + AC_DEFINE(HAVE_LIBS3,1,[Define to 1 if you have libs3]) + have_libs3=yes + ]) + ]) + ]) +fi + +AC_SUBST(S3_INC) +AC_SUBST(S3_LIBS) + AC_LANG_PUSH(C++) AC_CHECK_FUNCS(backtrace) AC_LANG_POP(C++) @@ -3227,7 +3322,6 @@ AC_ARG_WITH(systemd, PFILES="${PFILES} \ platforms/systemd/Makefile \ - platforms/systemd/bacula.conf \ platforms/systemd/bacula-dir.service \ platforms/systemd/bacula-fd.service \ platforms/systemd/bacula-sd.service" diff --git a/bacula/configure b/bacula/configure index 1e8702281b..a177592834 100755 --- a/bacula/configure +++ b/bacula/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for bacula 7.4.4. +# Generated by GNU Autoconf 2.69 for bacula 7.5.1. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -587,8 +587,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='bacula' PACKAGE_TARNAME='bacula' -PACKAGE_VERSION='7.4.4' -PACKAGE_STRING='bacula 7.4.4' +PACKAGE_VERSION='7.5.1' +PACKAGE_STRING='bacula 7.5.1' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -653,6 +653,8 @@ LZO_INC AFS_LIBS AFS_CFLAGS ZLIBS +S3_LIBS +S3_INC LIBOBJS X_EXTRA_LIBS X_LIBS @@ -880,6 +882,10 @@ LIBBACCATS_LT_RELEASE LIBBACSQL_LT_RELEASE LIBBACCFG_LT_RELEASE LIBBAC_LT_RELEASE +LIBRSYNC_VERSION +NDMP_VERSION +JAVA_VERSION +VIX_VERSION BQT4_VERSION DEPKGS_VERSION DEPKGS_QT_VERSION @@ -912,6 +918,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -1004,6 +1011,7 @@ with_embedded_mysql with_sqlite3 enable_largefile with_x +with_s3 enable_afs with_afsdir enable_lzo @@ -1064,6 +1072,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1316,6 +1325,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1453,7 +1471,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1566,7 +1584,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures bacula 7.4.4 to adapt to many kinds of systems. +\`configure' configures bacula 7.5.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1606,6 +1624,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1635,7 +1654,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of bacula 7.4.4:";; + short | recursive ) echo "Configuration of bacula 7.5.1:";; esac cat <<\_ACEOF @@ -1746,6 +1765,7 @@ Optional Packages: install directory, default is to search through a number of common places for the SQLite3 files. --with-x use the X Window System + --with-s3[=DIR] specify s3 library directory --with-afsdir[=DIR] Directory holding AFS includes/libs --with-lzo[=DIR] specify lzo library directory --with-systemd[=UNITDIR] @@ -1833,7 +1853,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -bacula configure 7.4.4 +bacula configure 7.5.1 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2768,7 +2788,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by bacula $as_me 7.4.4, which was +It was created by bacula $as_me 7.5.1, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3266,6 +3286,11 @@ BDB_VERSION=`sed -n -e 's/^#define BDB_VERSION \(.*\)$/\1/p' ${srcdir}/src/cats/ DEPKGS_VERSION=`sed -n -e 's/^#define DEPKGS_VERSION \(.*\)$/\1/p' ${srcdir}/src/cats/cats.h` DEPKGS_QT_VERSION=`sed -n -e 's/^#define DEPKGS_QT_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` BQT_VERSION=`sed -n -e 's/^#define BQT_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +VIX_VERSION=`sed -n -e 's/^#define VIX_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +JAVA_VERSION=`sed -n -e 's/^#define JAVA_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +NDMP_VERSION=`sed -n -e 's/^#define NDMP_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` +LIBRSYNC_VERSION=`sed -n -e 's/^#define LIBRSYNC_VERSION.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` + LIBBAC_LT_RELEASE=`sed -n -e 's/^#.*LIBBAC_LT_RELEASE.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` LIBBACCFG_LT_RELEASE=`sed -n -e 's/^#.*LIBBACCFG_LT_RELEASE.*"\(.*\)"$/\1/p' ${srcdir}/src/version.h` @@ -3287,7 +3312,6 @@ LIBBACFIND_LT_RELEASE=`sed -n -e 's/^#.*LIBBACFIND_LT_RELEASE.*"\(.*\)"$/\1/p' $ LIBBACFIND_LT_RELEASE=${LIBBACFIND_LT_RELEASE:-$VERSION} - PFILES="platforms/Makefile" echo "configuring for ${BACULA} $VERSION ($DATE)" @@ -22647,7 +22671,7 @@ _ACEOF fi done -for ac_func in lstat lchown lchmod futimes fchmod fchown +for ac_func in lstat lchown lchmod futimes fchmod fchown lutimes do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -26981,6 +27005,221 @@ fi +ac_fn_c_check_decl "$LINENO" "O_CLOEXEC" "ac_cv_have_decl_O_CLOEXEC" " +#ifdef HAVE_FCNTL_H +# include +#endif + +" +if test "x$ac_cv_have_decl_O_CLOEXEC" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_O_CLOEXEC $ac_have_decl +_ACEOF +if test $ac_have_decl = 1; then : + +else + +$as_echo "#define O_CLOEXEC 0" >>confdefs.h + +fi + + +ac_fn_c_check_decl "$LINENO" "FD_CLOEXEC" "ac_cv_have_decl_FD_CLOEXEC" " +#ifdef HAVE_FCNTL_H +# include +#endif + +" +if test "x$ac_cv_have_decl_FD_CLOEXEC" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_FD_CLOEXEC $ac_have_decl +_ACEOF +if test $ac_have_decl = 1; then : + +else + +$as_echo "#define FD_CLOEXEC 0" >>confdefs.h + +fi + + +ac_fn_c_check_decl "$LINENO" "SOCK_CLOEXEC" "ac_cv_have_decl_SOCK_CLOEXEC" " +#ifdef HAVE_SYS_SOCKET_H +# include +#endif + +" +if test "x$ac_cv_have_decl_SOCK_CLOEXEC" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_SOCK_CLOEXEC $ac_have_decl +_ACEOF +if test $ac_have_decl = 1; then : + +else + +$as_echo "#define SOCK_CLOEXEC 0" >>confdefs.h + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for close on exec modifier for fopen()" >&5 +$as_echo_n "checking for close on exec modifier for fopen()... " >&6; } +if ${ac_cv_feature_stream_cloexec_flag+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test $ac_cv_have_decl_O_CLOEXEC = yes ; then + if test $ac_cv_have_decl_SOCK_CLOEXEC = yes ; then + ac_cv_feature_stream_cloexec_flag="e" + fi + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_feature_stream_cloexec_flag" >&5 +$as_echo "$ac_cv_feature_stream_cloexec_flag" >&6; } + +if test "x$ac_cv_feature_stream_cloexec_flag" = "xe" ; then + +$as_echo "#define HAVE_STREAM_CLOEXEC 0" >>confdefs.h + +fi + + +cat >>confdefs.h <<_ACEOF +#define STREAM_CLOEXEC "$ac_cv_feature_stream_cloexec_flag" +_ACEOF + + +ac_fn_c_check_func "$LINENO" "accept4" "ac_cv_func_accept4" +if test "x$ac_cv_func_accept4" = xyes; then : + +$as_echo "#define HAVE_ACCEPT4 1" >>confdefs.h + +fi + + +S3_INC= +S3_LIBS= +S3_LDFLAGS= +have_libs3=no + +if test x$support_s3 = xyes; then + +# Check whether --with-s3 was given. +if test "${with_s3+set}" = set; then : + withval=$with_s3; + case "$with_s3" in + no) + : + ;; + yes|*) + if test -f ${with_s3}/include/libs3.h; then + S3_INC="-I${with_s3}/include" + S3_LDFLAGS="-L${with_s3}/lib" + with_s3="${with_s3}/include" + else + with_s3="/usr/include" + fi + + as_ac_Header=`$as_echo "ac_cv_header_${with_s3}/libs3.h" | $as_tr_sh` +ac_fn_c_check_header_mongrel "$LINENO" "${with_s3}/libs3.h" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : + + +$as_echo "#define HAVE_LIBS3 1" >>confdefs.h + + S3_LIBS="${S3_LDFLAGS} -ls3" + have_libs3="yes" + +else + + echo " " + echo "libs3.h not found. s3 turned off ..." + echo " " + + +fi + + + ;; + esac + +else + + ac_fn_c_check_header_mongrel "$LINENO" "libs3.h" "ac_cv_header_libs3_h" "$ac_includes_default" +if test "x$ac_cv_header_libs3_h" = xyes; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for S3_initialize in -ls3" >&5 +$as_echo_n "checking for S3_initialize in -ls3... " >&6; } +if ${ac_cv_lib_s3_S3_initialize+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ls3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char S3_initialize (); +int +main () +{ +return S3_initialize (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_s3_S3_initialize=yes +else + ac_cv_lib_s3_S3_initialize=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_s3_S3_initialize" >&5 +$as_echo "$ac_cv_lib_s3_S3_initialize" >&6; } +if test "x$ac_cv_lib_s3_S3_initialize" = xyes; then : + + S3_LIBS="-ls3" + +$as_echo "#define HAVE_LIBS3 1" >>confdefs.h + + have_libs3=yes + +fi + + +fi + + + +fi + +fi + + + + ac_ext=cpp ac_cpp='$CXXCPP $CPPFLAGS' ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' @@ -30885,7 +31124,6 @@ if test "${with_systemd+set}" = set; then : PFILES="${PFILES} \ platforms/systemd/Makefile \ - platforms/systemd/bacula.conf \ platforms/systemd/bacula-dir.service \ platforms/systemd/bacula-fd.service \ platforms/systemd/bacula-sd.service" @@ -31460,7 +31698,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by bacula $as_me 7.4.4, which was +This file was extended by bacula $as_me 7.5.1, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -31526,7 +31764,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -bacula config.status 7.4.4 +bacula config.status 7.5.1 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/bacula/platforms/redhat/bacula-dir.in b/bacula/platforms/redhat/bacula-dir.in index a36d4a4615..edbf1ea319 100755 --- a/bacula/platforms/redhat/bacula-dir.in +++ b/bacula/platforms/redhat/bacula-dir.in @@ -60,14 +60,20 @@ case "$1" in daemon $DAEMON_OPTIONS @sbindir@/bacula-dir $2 ${DIR_OPTIONS} -c @sysconfdir@/bacula-dir.conf RETVAL=$? echo - [ $RETVAL -eq 0 ] && touch @subsysdir@/bacula-dir + if [ $RETVAL -eq 0 ]; then + touch @subsysdir@/bacula-dir + logger -p daemon.info "bacula-dir started" >/dev/null 2>/dev/null + fi ;; stop) echo -n "Stopping Bacula Director services: " killproc @sbindir@/bacula-dir RETVAL=$? echo - [ $RETVAL -eq 0 ] && rm -f @subsysdir@/bacula-dir + if [ $RETVAL -eq 0 ]; then + rm -f @subsysdir@/bacula-dir + logger -p daemon.info "bacula-dir stopped" >/dev/null 2>/dev/null + fi ;; restart) $0 stop diff --git a/bacula/platforms/redhat/bacula-sd.in b/bacula/platforms/redhat/bacula-sd.in index d8c3440bcc..e8abce3f3a 100755 --- a/bacula/platforms/redhat/bacula-sd.in +++ b/bacula/platforms/redhat/bacula-sd.in @@ -61,14 +61,20 @@ case "$1" in daemon $DAEMON_OPTIONS @sbindir@/bacula-sd $2 ${SD_OPTIONS} -c @sysconfdir@/bacula-sd.conf RETVAL=$? echo - [ $RETVAL -eq 0 ] && touch @subsysdir@/bacula-sd + if [ $RETVAL -eq 0 ]; then + touch @subsysdir@/bacula-sd + logger -p daemon.info "bacula-sd started" >/dev/null 2>/dev/null + fi ;; stop) echo -n "Stopping Bacula Storage services: " killproc @sbindir@/bacula-sd RETVAL=$? echo - [ $RETVAL -eq 0 ] && rm -f @subsysdir@/bacula-sd + if [ $RETVAL -eq 0 ]; then + rm -f @subsysdir@/bacula-sd + logger -p daemon.info "bacula-sd stopped" >/dev/null 2>/dev/null + fi ;; restart) $0 stop diff --git a/bacula/platforms/systemd/bacula-sd.service.in b/bacula/platforms/systemd/bacula-sd.service.in index 6de48ba804..6f909bdb65 100644 --- a/bacula/platforms/systemd/bacula-sd.service.in +++ b/bacula/platforms/systemd/bacula-sd.service.in @@ -27,8 +27,7 @@ Group=@sd_group@ ExecStart=@sbindir@/bacula-sd -c @sysconfdir@/bacula-sd.conf PIDFile=@piddir@/bacula-sd.@sd_port@.pid StandardError=syslog -#LimitMEMLOCK=infinity -TimeoutStopSec=3min +LimitMEMLOCK=infinity [Install] WantedBy=multi-user.target diff --git a/bacula/release/bgit.py b/bacula/release/bgit.py new file mode 100755 index 0000000000..66f823ad95 --- /dev/null +++ b/bacula/release/bgit.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python2 +# this program compare two branche of GIT +# and show the differences +import sys +import os +import logging +import collections +import re +import argparse +import time + +try: + import git +except ImportError: + print >>sys.stderr, "you must install python-git aka GitPython" + sys.exit(1) + + +def add_console_logger(): + console=logging.StreamHandler() + console.setFormatter(logging.Formatter('%(levelname)-3.3s %(filename)s:%(lineno)d %(message)s', '%H:%M:%S')) + console.setLevel(logging.DEBUG) # must be INFO for prod + logging.getLogger().addHandler(console) + return console + +def add_file_logger(filename): + filelog=logging.FileHandler(filename) + # %(asctime)s '%Y-%m-%d %H:%M:%S' + filelog.setFormatter(logging.Formatter('%(asctime)s %(levelname)-3.3s %(filename)s:%(lineno)d %(message)s', '%H:%M:%S')) + filelog.setLevel(logging.DEBUG) + logging.getLogger().addHandler(filelog) + return filelog + + + +def run_cmp_branch(repo, args): + print args.branch1 + print args.branch2 +# for commit in repo.iter_commits(args.branch1, max_count=10): +# print commit.hexsha, commit.committed_date, commit.author.name, commit.message + +# print dir(repo) + commons=repo.merge_base(args.branch1, args.branch2) + if len(commons)!=1: + print "cannot find the unique common commit between", args.branch1, args.branch2 + + common=commons[0] + # make a list of all know commit in branch-2 + commits2=set() + for commit in repo.iter_commits(args.branch2): + if commit.hexsha==common.hexsha: + break + + subject=commit.message.split('\n', 1)[0] + commits2.add((commit.authored_date, commit.author.name, subject)) + #print commit.committed_date, commit.author.name, subject + + # list and compare with commits of branch-& + for commit in repo.iter_commits(args.branch1): + if commit.hexsha==common.hexsha: + break + subject=commit.message.split('\n', 1)[0] + date=time.strftime("%Y-%m-%d %H:%M", time.gmtime(commit.authored_date)) + if (commit.authored_date, commit.author.name, subject) in commits2: + print "=", date, commit.author.name, subject + else: + print "+", date, commit.author.name, subject + +mainparser=argparse.ArgumentParser(description='git utility for bacula') +subparsers=mainparser.add_subparsers(dest='command', metavar='', title='valid commands') + +git_parser=argparse.ArgumentParser(add_help=False) +git_parser.add_argument('--git_dir', metavar='GIT-DIR', type=str, default='.', help='the directory with the .git sub dir') + +parser=subparsers.add_parser('cmp_branch', parents=[git_parser, ], help='compare two branches, highligh commits missing in the second branch') + +parser.add_argument('branch1', metavar='BRANCH-1', help='the first branch') +parser.add_argument('branch2', metavar='BRANCH-2', help='the second branch') + +args=mainparser.parse_args() + + +logging.getLogger().setLevel(logging.DEBUG) + +add_console_logger() +print args.git_dir +print "logging into gitstat.log" +add_file_logger('gitstat.log') + +# search the git repo +repo=None +if args.git_dir: + if args.git_dir=='.': + path=os.getcwd() + while path and not os.path.isdir(os.path.join(path, '.git')): + path=os.path.dirname(path) + print path + + if path and os.path.isdir(os.path.join(path, '.git')): + try: + repo=git.Repo(path) + except git.exc.InvalidGitRepositoryError: + parser.error("git repository not found in %s" % (path,)) + else: + args.git_dir=path + else: + parser.error("not .git directory found above %s" % (os.getcwd(),)) + + else: + try: + repo=git.Repo(args.git_dir) + except git.exc.InvalidGitRepositoryError: + parser.error("git repository not found in %s" % (args.git_dir,)) + +if args.command=='cmp_branch': + run_cmp_branch(repo, args) + diff --git a/bacula/scripts/Makefile.in b/bacula/scripts/Makefile.in index e9b529d78d..649ce9bc0c 100755 --- a/bacula/scripts/Makefile.in +++ b/bacula/scripts/Makefile.in @@ -39,6 +39,7 @@ install: installdirs $(INSTALL_SCRIPT) bacula $(DESTDIR)$(scriptdir)/bacula $(INSTALL_SCRIPT) bacula_config $(DESTDIR)$(scriptdir)/bacula_config $(INSTALL_SCRIPT) bacula $(DESTDIR)$(sbindir)/bacula + $(INSTALL_SCRIPT) tapealert $(DESTDIR)$(scriptdir)/tapealert $(INSTALL_SCRIPT) bacula-ctl-dir $(DESTDIR)$(scriptdir)/bacula-ctl-dir $(INSTALL_SCRIPT) bacula-ctl-fd $(DESTDIR)$(scriptdir)/bacula-ctl-fd $(INSTALL_SCRIPT) bacula-ctl-sd $(DESTDIR)$(scriptdir)/bacula-ctl-sd @@ -58,11 +59,6 @@ install: installdirs $(MV) -f ${DESTDIR}${scriptdir}/disk-changer ${DESTDIR}${scriptdir}/disk-changer.old; \ fi $(INSTALL_SCRIPT) disk-changer $(DESTDIR)$(scriptdir)/disk-changer - @if test -f ${DESTDIR}${scriptdir}/dvd-handler; then \ - echo " ==> Saving existing dvd-handler to dvd-handler.old"; \ - $(MV) -f ${DESTDIR}${scriptdir}/dvd-handler ${DESTDIR}${scriptdir}/dvd-handler.old; \ - fi - $(INSTALL_SCRIPT) dvd-handler $(DESTDIR)$(scriptdir)/dvd-handler $(INSTALL_DATA) btraceback.gdb $(DESTDIR)$(scriptdir)/btraceback.gdb $(INSTALL_DATA) btraceback.dbx $(DESTDIR)$(scriptdir)/btraceback.dbx $(INSTALL_DATA) btraceback.mdb $(DESTDIR)$(scriptdir)/btraceback.mdb @@ -77,6 +73,7 @@ uninstall: (cd $(DESTDIR)$(scriptdir); $(RMF) bacula) (cd $(DESTDIR)$(scriptdir); $(RMF) bacula_config) (cd $(DESTDIR)$(sbindir); $(RMF) bacula) + (cd $(DESTDIR)$(sbindir); $(RMF) tapealert) (cd $(DESTDIR)$(scriptdir); $(RMF) bacula-ctl-dir) (cd $(DESTDIR)$(scriptdir); $(RMF) bacula-ctl-fd) (cd $(DESTDIR)$(scriptdir); $(RMF) bacula-ctl-sd) @@ -95,13 +92,13 @@ Makefile: Makefile.in && CONFIG_FILES=$(thisdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status chmod 755 bacula btraceback chmod 755 bacula-ctl-dir bacula-ctl-fd bacula-ctl-sd bacula_config - chmod 755 disk-changer mtx-changer dvd-handler bconsole + chmod 755 disk-changer mtx-changer bconsole tapealert Makefiles: $(SHELL) config.status chmod 755 bacula btraceback chmod 755 bacula-ctl-dir bacula-ctl-fd bacula-ctl-sd - chmod 755 mtx-changer dvd-handler bconsole + chmod 755 mtx-changer bconsole tapealert clean: @$(RMF) *~ 1 2 3 diff --git a/bacula/scripts/tapealert b/bacula/scripts/tapealert new file mode 100755 index 0000000000..c8c68c0adb --- /dev/null +++ b/bacula/scripts/tapealert @@ -0,0 +1,68 @@ +#!/bin/sh +# +# Copyright (C) 2000-2016 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Bacula interface to tapeinfo to get tape alerts +# +# tapealert %l (control device name) +# +# Note: you must have in your SD Device resource: +# Alert Command = /full-path/tapealert %l +# Control Device = /dev/sg0n (where this is the scsi control +# device for the device you are using). +# + +# Note: to test +# 1. uncomment out the DEBUG=1 line below +# 2. Possibly remove or add TapeAlert[nn]: that you want to test. +# Note, the message following the : is not used. +# 3. Run Bacula +# +#DEBUG=1 + +tapeinfo=`which tapeinfo` + +if [ x${tapeinfo} = x ] ; then + echo "tapeinfo program not found, but is required." + exit 1 +fi +if [ x$1 = x ] ; then + echo "First argument missing. Must be device control name." + exit 1 +fi + + +if [ x$DEBUG = x ] ; then +$tapeinfo -f $1 |grep "^TapeAlert" - |cut -b1-13 +exit $? + +else + +# For testing only +cat <inc_use_count(); db = mdb; /* need to inc ref count */ @@ -55,14 +61,18 @@ Bvfs::Bvfs(JCR *j, BDB *mdb) { tmp = get_pool_memory(PM_NAME); escaped_list = get_pool_memory(PM_NAME); *filename = *jobids = *prev_dir = *pattern = 0; - dir_filenameid = pwd_id = offset = 0; + pwd_id = offset = 0; see_copies = see_all_versions = false; + compute_delta = true; limit = 1000; attr = new_attr(jcr); list_entries = result_handler; user_data = this; username = NULL; job_acl = client_acl = pool_acl = fileset_acl = NULL; + last_dir_acl = NULL; + dir_acl = NULL; + use_acl = false; } Bvfs::~Bvfs() { @@ -77,6 +87,9 @@ Bvfs::~Bvfs() { } free_attr(attr); jcr->dec_use_count(); + if (dir_acl) { + delete dir_acl; + } } char *Bvfs::escape_list(alist *lst) @@ -113,7 +126,8 @@ char *Bvfs::escape_list(alist *lst) return escaped_list; } -void Bvfs::filter_jobid() +/* Returns the number of jobids in the result */ +int Bvfs::filter_jobid() { POOL_MEM query; POOL_MEM sub_where; @@ -122,7 +136,14 @@ void Bvfs::filter_jobid() /* No ACL, no username, no check */ if (!job_acl && !fileset_acl && !client_acl && !pool_acl && !username) { Dmsg0(dbglevel_sql, "No ACL\n"); - return; + /* Just count the number of items in the list */ + int nb = (*jobids != 0) ? 1 : 0; + for (char *p=jobids; *p ; p++) { + if (*p == ',') { + nb++; + } + } + return nb; } if (job_acl) { @@ -172,18 +193,21 @@ void Bvfs::filter_jobid() Dmsg1(dbglevel_sql, "q=%s\n", query.c_str()); db->bdb_sql_query(query.c_str(), db_list_handler, &ctx); pm_strcpy(jobids, ctx.list); + return ctx.count; } -void Bvfs::set_jobid(JobId_t id) +/* Return the number of jobids after the filter */ +int Bvfs::set_jobid(JobId_t id) { Mmsg(jobids, "%lld", (uint64_t)id); - filter_jobid(); + return filter_jobid(); } -void Bvfs::set_jobids(char *ids) +/* Return the number of jobids after the filter */ +int Bvfs::set_jobids(char *ids) { pm_strcpy(jobids, ids); - filter_jobid(); + return filter_jobid(); } /* @@ -218,7 +242,7 @@ public: nb_node = 0; table_node = New(alist(5, owned_by_alist)); table_node->append(nodes); - } + }; hlink *get_hlink() { if (++nb_node >= max_node) { @@ -227,23 +251,23 @@ public: table_node->append(nodes); } return nodes + nb_node; - } + }; bool lookup(char *pathid) { bool ret = cache_ppathid->lookup(pathid) != NULL; return ret; - } + }; void insert(char *pathid) { hlink *h = get_hlink(); cache_ppathid->insert(pathid, h); - } + }; ~pathid_cache() { cache_ppathid->destroy(); free(cache_ppathid); delete table_node; - } + }; private: pathid_cache(const pathid_cache &); /* prohibit pass by value */ pathid_cache &operator= (const pathid_cache &);/* prohibit class assignment*/ @@ -471,7 +495,6 @@ static int update_path_hierarchy_cache(JCR *jcr, free(result); } - if (mdb->bdb_get_type_index() == SQL_TYPE_SQLITE3) { Mmsg(mdb->cmd, "INSERT INTO PathVisibility (PathId, JobId) " @@ -530,21 +553,6 @@ bail_out: return ret; } -/* - * Find an store the filename descriptor for empty directories Filename.Name='' - */ -DBId_t Bvfs::get_dir_filenameid() -{ - uint32_t id; - if (dir_filenameid) { - return dir_filenameid; - } - Mmsg(db->cmd, "SELECT FilenameId FROM Filename WHERE Name = ''"); - db_sql_query(db, db->cmd, db_int_handler, &id); - dir_filenameid = id; - return dir_filenameid; -} - /* Compute the cache for the bfileview compoment */ void Bvfs::fv_update_cache() { @@ -577,6 +585,21 @@ void Bvfs::fv_update_cache() db->bdb_unlock(); } +/* + * Find an store the filename descriptor for empty directories Filename.Name='' + */ +DBId_t Bvfs::get_dir_filenameid() +{ + uint32_t id; + if (dir_filenameid) { + return dir_filenameid; + } + Mmsg(db->cmd, "SELECT FilenameId FROM Filename WHERE Name = ''"); + db_sql_query(db, db->cmd, db_int_handler, &id); + dir_filenameid = id; + return dir_filenameid; +} + /* Not yet working */ void Bvfs::fv_get_big_files(int64_t pathid, int64_t min_size, int32_t limit) { @@ -593,6 +616,7 @@ void Bvfs::fv_get_big_files(int64_t pathid, int64_t min_size, int32_t limit) "LIMIT %d ", pathid, jobids, min_size, limit); } + /* Get the current path size and files count */ void Bvfs::fv_get_current_size_and_count(int64_t pathid, int64_t *size, int64_t *count) { @@ -828,12 +852,22 @@ void Bvfs::update_cache() bvfs_update_path_hierarchy_cache(jcr, db, jobids); } + +bool Bvfs::ch_dir(DBId_t pathid) +{ + reset_offset(); + + pwd_id = pathid; + return pwd_id != 0; +} + + /* Change the current directory, returns true if the path exists */ bool Bvfs::ch_dir(const char *path) { + db->bdb_lock(); pm_strcpy(db->path, path); db->pnl = strlen(db->path); - db->bdb_lock(); ch_dir(db->bdb_get_path_record(jcr)); db->bdb_unlock(); return pwd_id != 0; @@ -880,6 +914,108 @@ void Bvfs::get_all_file_versions(DBId_t pathid, FileId_t fnid, const char *clien db->bdb_sql_query(query.c_str(), list_entries, user_data); } +/* + * Get all file versions for a specified client + * TODO: Handle basejobs using different client + */ +bool Bvfs::get_delta(FileId_t fileid) +{ + Dmsg1(dbglevel, "get_delta(%lld)\n", (uint64_t)fileid); + char ed1[50]; + int32_t num; + SQL_ROW row; + POOL_MEM q; + POOL_MEM query; + char *fn = NULL; + bool ret = false; + db->bdb_lock(); + + /* Check if some FileId have DeltaSeq > 0 + * Foreach of them we need to get the accurate_job list, and compute + * what are dependencies + */ + Mmsg(query, + "SELECT F.JobId, FN.Name, F.PathId, F.DeltaSeq " + "FROM File AS F, Filename AS FN WHERE FileId = %lld " + "AND FN.FilenameId = F.FilenameId AND DeltaSeq > 0", fileid); + + if (!db->QueryDB(jcr, query.c_str())) { + Dmsg1(dbglevel_sql, "Can't execute query=%s\n", query.c_str()); + goto bail_out; + } + + /* TODO: Use an other DB connection can avoid to copy the result of the + * previous query into a temporary buffer + */ + num = db->sql_num_rows(); + Dmsg2(dbglevel, "Found %d Delta parts q=%s\n", + num, query.c_str()); + + if (num > 0 && (row = db->sql_fetch_row())) { + JOB_DBR jr, jr2; + db_list_ctx lst; + memset(&jr, 0, sizeof(jr)); + memset(&jr2, 0, sizeof(jr2)); + + fn = bstrdup(row[1]); /* Filename */ + int64_t jid = str_to_int64(row[0]); /* JobId */ + int64_t pid = str_to_int64(row[2]); /* PathId */ + + /* Need to limit the query to StartTime, Client/Fileset */ + jr2.JobId = jid; + if (!db->bdb_get_job_record(jcr, &jr2)) { + Dmsg1(0, "Unable to get job record for jobid %d\n", jid); + goto bail_out; + } + + jr.JobId = jid; + jr.ClientId = jr2.ClientId; + jr.FileSetId = jr2.FileSetId; + jr.JobLevel = L_INCREMENTAL; + jr.StartTime = jr2.StartTime; + + /* Get accurate jobid list */ + if (!db->bdb_get_accurate_jobids(jcr, &jr, &lst)) { + Dmsg1(0, "Unable to get Accurate list for jobid %d\n", jid); + goto bail_out; + } + + /* Escape filename */ + db->fnl = strlen(fn); + db->esc_name = check_pool_memory_size(db->esc_name, 2*db->fnl+2); + db->bdb_escape_string(jcr, db->esc_name, fn, db->fnl); + + edit_int64(pid, ed1); /* pathid */ + + int id=db->bdb_get_type_index(); + Mmsg(query, bvfs_select_delta_version_with_basejob_and_delta[id], + lst.list, db->esc_name, ed1, + lst.list, db->esc_name, ed1, + lst.list, lst.list); + + Mmsg(db->cmd, + // 0 1 2 3 4 5 6 7 + "SELECT 'd', PathId, 0, JobId, LStat, FileId, DeltaSeq, JobTDate" + " FROM (%s) AS F1 " + "ORDER BY DeltaSeq ASC", + query.c_str()); + + Dmsg1(dbglevel_sql, "q=%s\n", db->cmd); + + if (!db->bdb_sql_query(db->cmd, list_entries, user_data)) { + Dmsg1(dbglevel_sql, "Can't exec q=%s\n", db->cmd); + goto bail_out; + } + } + ret = true; +bail_out: + if (fn) { + free(fn); + } + db->bdb_unlock(); + return ret; +} + /* * Get all volumes for a specific file */ @@ -925,6 +1061,12 @@ int Bvfs::_handle_path(void *ctx, int fields, char **row) /* can have the same path 2 times */ if (strcmp(row[BVFS_PathId], prev_dir)) { pm_strcpy(prev_dir, row[BVFS_PathId]); + if (strcmp(NPRTB(row[BVFS_FileIndex]), "0") == 0 && + strcmp(NPRTB(row[BVFS_FileId]), "0") != 0) + { + /* The directory was probably deleted */ + return 0; + } return list_entries(user_data, fields, row); } } @@ -1210,9 +1352,10 @@ bool Bvfs::compute_restore_list(char *fileid, char *dirid, char *hardlink, if (*fileid) { /* Select files with their direct id */ init=true; - Mmsg(tmp,"SELECT Job.JobId, JobTDate, FileIndex, FilenameId, " + Mmsg(tmp,"SELECT Job.JobId, JobTDate, FileIndex, Filename.Name, " "PathId, FileId " - "FROM File JOIN Job USING (JobId) WHERE FileId IN (%s)", + "FROM File,Job,Filename WHERE Job.JobId=File.Jobid " + "AND File.FilenameId=Filename.FilenameId AND FileId IN (%s)", fileid); pm_strcat(query, tmp.c_str()); } @@ -1388,7 +1531,7 @@ bail_out: void Bvfs::insert_missing_delta(char *output_table, int64_t *res) { - char ed1[50], ed2[50]; + char ed1[50]; db_list_ctx lst; POOL_MEM query; JOB_DBR jr, jr2; @@ -1422,13 +1565,17 @@ void Bvfs::insert_missing_delta(char *output_table, int64_t *res) Dmsg1(dbglevel_sql, "JobId list after strip is %s\n", lst.list); - edit_int64(res[2], ed1); /* fnid */ - edit_int64(res[3], ed2); /* pathid */ + /* Escape filename */ + db->fnl = strlen((char *)res[2]); + db->esc_name = check_pool_memory_size(db->esc_name, 2*db->fnl+2); + db->bdb_escape_string(jcr, db->esc_name, (char *)res[2], db->fnl); + + edit_int64(res[3], ed1); /* pathid */ int id=db->bdb_get_type_index(); Mmsg(query, bvfs_select_delta_version_with_basejob_and_delta[id], - lst.list, ed1, ed2, - lst.list, ed1, ed2, + lst.list, db->esc_name, ed1, + lst.list, db->esc_name, ed1, lst.list, lst.list); Mmsg(db->cmd, "INSERT INTO %s " @@ -1439,5 +1586,5 @@ void Bvfs::insert_missing_delta(char *output_table, int64_t *res) Dmsg1(dbglevel_sql, "Can't exec q=%s\n", db->cmd); } } - + #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL */ diff --git a/bacula/src/cats/bvfs.h b/bacula/src/cats/bvfs.h index 5eb8f0ae4b..f75006ddce 100644 --- a/bacula/src/cats/bvfs.h +++ b/bacula/src/cats/bvfs.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ - #ifndef __BVFS_H_ #define __BVFS_H_ 1 @@ -38,24 +37,32 @@ typedef enum { BVFS_FILE_RECORD = 'F', BVFS_DIR_RECORD = 'D', BVFS_FILE_VERSION = 'V', - BVFS_VOLUME_LIST = 'L' + BVFS_VOLUME_LIST = 'L', + BVFS_DELTA_RECORD = 'd', } bvfs_handler_type; typedef enum { BVFS_Type = 0, /* Could be D, F, V, L */ - BVFS_PathId = 1, - BVFS_FilenameId = 2, + BVFS_PathId = 1, + BVFS_FilenameId = 5, /* No longer in use, use fileid instead */ + + BVFS_Name = 2, + BVFS_JobId = 3, - BVFS_Name = 3, - BVFS_JobId = 4, + BVFS_LStat = 4, /* Can be empty for missing directories */ + BVFS_FileId = 5, /* Can be empty for missing directories */ - BVFS_LStat = 5, /* Can be empty for missing directories */ - BVFS_FileId = 6, /* Can be empty for missing directories */ + /* Only if Path record */ + BVFS_FileIndex = 6, /* Only if File Version record */ - BVFS_Md5 = 3, + BVFS_Md5 = 6, BVFS_VolName = 7, - BVFS_VolInchanger = 8 + BVFS_VolInchanger = 8, + + /* Only if Delta record */ + BVFS_DeltaSeq = 6, + BVFS_JobTDate = 7 } bvfs_row_index; class Bvfs { @@ -64,8 +71,13 @@ public: Bvfs(JCR *j, BDB *mdb); virtual ~Bvfs(); - void set_jobid(JobId_t id); - void set_jobids(char *ids); + void set_compute_delta(bool val) { + compute_delta = val; + }; + + /* Return the number of jobids after the filter */ + int set_jobid(JobId_t id); + int set_jobids(char *ids); char *get_jobids() { return jobids; @@ -97,10 +109,7 @@ public: /* It's much better to access Path though their PathId, it * avoids mistakes with string encoding */ - void ch_dir(DBId_t pathid) { - reset_offset(); - pwd_id = pathid; - } + bool ch_dir(DBId_t pathid); /* * Returns true if the directory exists @@ -125,13 +134,15 @@ public: see_copies = val; } - void filter_jobid(); /* Call after set_username */ + DBId_t get_dir_filenameid(); + + int filter_jobid(); /* Call after set_username, returns the number of jobids */ void set_username(char *user) { if (user) { username = bstrdup(user); } - } + }; char *escape_list(alist *list); @@ -143,30 +154,33 @@ public: return false; } return true; - } + }; /* Keep a pointer to various ACLs */ void set_job_acl(alist *lst) { job_acl = copy_acl(lst)?lst:NULL; - } + use_acl = true; + }; void set_fileset_acl(alist *lst) { fileset_acl = copy_acl(lst)?lst:NULL; - } + use_acl = true; + }; void set_client_acl(alist *lst) { client_acl = copy_acl(lst)?lst:NULL; - } + use_acl = true; + }; void set_pool_acl(alist *lst) { pool_acl = copy_acl(lst)?lst:NULL; - } - + use_acl = true; + }; void set_handler(DB_RESULT_HANDLER *h, void *ctx) { list_entries = h; user_data = ctx; - } + }; DBId_t get_pwd() { return pwd_id; - } + }; ATTR *get_attr() { return attr; @@ -203,6 +217,21 @@ public: /* Get a list of volumes */ void get_volumes(FileId_t fileid); + /* Get Delta parts of a file */ + bool get_delta(FileId_t fileid); + + /* Check if the parent directories are accessible */ + bool check_path_access(DBId_t pathid); + + /* Check if the full path is authorized by the current set of ACLs */ + bool check_full_path_access(int nb, sellist *sel, db_list_ctx *toexcl); + + alist *dir_acl; + + int check_dirs; /* When it's 1, we check the against directory_acl */ + bool can_access(struct stat *st); + bool can_access_dir(const char *path); + private: Bvfs(const Bvfs &); /* prohibit pass by value */ Bvfs & operator = (const Bvfs &); /* prohibit class assignment */ @@ -224,6 +253,7 @@ private: alist *client_acl; alist *fileset_acl; alist *pool_acl; + char *last_dir_acl; ATTR *attr; /* Can be use by handler to call decode_stat() */ @@ -235,8 +265,11 @@ private: bool see_all_versions; bool see_copies; + bool compute_delta; - DBId_t get_dir_filenameid(); + db_list_ctx fileid_to_delete; /* used also by check_path_access */ + bool need_to_check_permissions(); + bool use_acl; /* bfileview */ void fv_get_big_files(int64_t pathid, int64_t min_size, int32_t limit); @@ -253,6 +286,7 @@ private: #define bvfs_is_file(row) ((row)[BVFS_Type][0] == BVFS_FILE_RECORD) #define bvfs_is_version(row) ((row)[BVFS_Type][0] == BVFS_FILE_VERSION) #define bvfs_is_volume_list(row) ((row)[BVFS_Type][0] == BVFS_VOLUME_LIST) +#define bvfs_is_delta_list(row) ((row)[BVFS_Type][0] == BVFS_DELTA_RECORD) void bvfs_update_fv_cache(JCR *jcr, BDB *mdb, char *jobids); int bvfs_update_path_hierarchy_cache(JCR *jcr, BDB *mdb, char *jobids); @@ -265,5 +299,4 @@ char *bvfs_parent_dir(char *path); */ char *bvfs_basename_dir(char *path); - #endif /* __BVFS_H_ */ diff --git a/bacula/src/cats/cats.h b/bacula/src/cats/cats.h index 2e03c40d47..acd88ff22e 100644 --- a/bacula/src/cats/cats.h +++ b/bacula/src/cats/cats.h @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -48,7 +48,7 @@ */ /* Current database version number for all drivers */ -#define BDB_VERSION 15 +#define BDB_VERSION 16 typedef void (DB_LIST_HANDLER)(void *, const char *); typedef int (DB_RESULT_HANDLER)(void *, int, char **); @@ -158,10 +158,14 @@ struct JOB_DBR { char cEndTime[MAX_TIME_LENGTH]; char cRealEndTime[MAX_TIME_LENGTH]; /* Extra stuff not in DB */ - int order; /* 0 ASC, 1 DESC */ - int limit; /* limit records to display */ + int order; /* 0 ASC, 1 DESC */ + int limit; /* limit records to display */ faddr_t rec_addr; uint32_t FileIndex; /* added during Verify */ + + int CorrNbJob; /* used by dbd_get_job_statistics() */ + int CorrJobBytes; /* used by dbd_get_job_statistics() */ + int CorrJobFiles; /* used by dbd_get_job_statistics() */ }; /* Job Media information used to create the media records @@ -262,6 +266,7 @@ struct POOL_DBR { int32_t Recycle; /* default Vol recycle flag */ uint32_t ActionOnPurge; /* action on purge, e.g. truncate the disk volume */ utime_t VolRetention; /* retention period in seconds */ + utime_t CacheRetention; /* cache retention period in seconds */ utime_t VolUseDuration; /* time in secs volume can be used */ uint32_t MaxVolJobs; /* Max Jobs on Volume */ uint32_t MaxVolFiles; /* Max files on Volume */ @@ -315,7 +320,7 @@ public: MEDIA_DBR() { memset(this, 0, sizeof(MEDIA_DBR)); }; ~MEDIA_DBR() { }; void clear() { memset(this, 0, sizeof(MEDIA_DBR)); }; - void copy(MEDIA_DBR *omr) { memcpy(this, omr, sizeof(MEDIA_DBR)); }; + void copy(MEDIA_DBR *omr) { memcpy(this, omr, sizeof(MEDIA_DBR)); sid_group = NULL; }; DBId_t MediaId; /* Unique volume id */ char VolumeName[MAX_NAME_LENGTH]; /* Volume name */ @@ -329,6 +334,8 @@ public: uint32_t VolJobs; /* number of jobs on this medium */ uint32_t VolFiles; /* Number of files */ uint32_t VolBlocks; /* Number of blocks */ + uint32_t VolParts; /* Number of cache parts */ + uint32_t VolCloudParts; /* Number of cloud parts */ uint32_t VolMounts; /* Number of times mounted */ uint32_t VolErrors; /* Number of read/write errors */ uint64_t VolWrites; /* Number of writes */ @@ -340,9 +347,11 @@ public: uint32_t VolType; /* Device type of where Volume labeled */ uint64_t MaxVolBytes; /* Max bytes to write to Volume */ uint64_t VolCapacityBytes; /* capacity estimate */ + uint64_t LastPartBytes; /* Bytes in last part */ uint64_t VolReadTime; /* time spent reading volume */ uint64_t VolWriteTime; /* time spent writing volume */ utime_t VolRetention; /* Volume retention in seconds */ + utime_t CacheRetention; /* Cache retention period in second */ utime_t VolUseDuration; /* time in secs volume can be used */ uint32_t ActionOnPurge; /* action on purge, e.g. truncate the disk volume */ uint32_t MaxVolJobs; /* Max Jobs on Volume */ @@ -370,6 +379,8 @@ public: char cLabelDate[MAX_TIME_LENGTH]; /* LabelData returned from DB */ char cInitialWrite[MAX_TIME_LENGTH]; /* InitialWrite returned from DB */ char *exclude_list; /* Optionnal exclude list for db_find_next_volume() */ + char *sid_group; /* Storageid group string */ + char sid[30]; /* edited StorageId */ bool set_first_written; bool set_label_date; }; diff --git a/bacula/src/cats/create_postgresql_database.in b/bacula/src/cats/create_postgresql_database.in index 940ee965df..ac15d524e8 100644 --- a/bacula/src/cats/create_postgresql_database.in +++ b/bacula/src/cats/create_postgresql_database.in @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # # shell script to create Bacula database(s) diff --git a/bacula/src/cats/grant_mysql_privileges.in b/bacula/src/cats/grant_mysql_privileges.in index f19eac11a2..53bb03ef0a 100644 --- a/bacula/src/cats/grant_mysql_privileges.in +++ b/bacula/src/cats/grant_mysql_privileges.in @@ -2,7 +2,7 @@ # # shell script to grant privileges to the bacula database # -# Copyright (C) 2000-2016 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # db_user=${db_user:-@db_user@} @@ -17,7 +17,7 @@ if [ "$db_ssl_options" != "" ]; then ssl_options="require $db_ssl_options" fi -if $bindir/mysql $* -u root -f 2>/dev/null 1>/dev/null </dev/null 1>/dev/null < '$wd/$args{db_name}.sql'"); print "Error while executing postgres dump $!\n"; - return 1; # in case of error + return 1; # in case of error } sub analyse_pgsql @@ -80,7 +82,7 @@ sub analyse_pgsql my $exitcode = $? >> 8; print grep { !/^WARNING:\s+skipping\s\"(pg_|sql_)/ } @output; if ($exitcode != 0) { - print "Error while executing postgres analyse. Exitcode=$exitcode\n"; + print "Error while executing postgres analyse. Exitcode=$exitcode\n"; } return $exitcode; } @@ -91,12 +93,12 @@ sub setup_env_mysql umask(0077); unlink("$wd/.my.cnf"); open(MY, ">$wd/.my.cnf") - or die "Can't open $wd/.my.cnf for writing $@"; + or die "Can't open $wd/.my.cnf for writing $@"; $args{db_address} = $args{db_address} || "localhost"; my $addr = "host=$args{db_address}"; - if ($args{db_socket}) { # unix socket is fastest than net socket - $addr = "socket=\"$args{db_socket}\""; + if ($args{db_socket}) { # unix socket is fastest than net socket + $addr = "socket=\"$args{db_socket}\""; } my $mode = $args{mode} || 'client'; print MY "[$mode] @@ -136,26 +138,26 @@ sub handle_catalog { my ($mode, %args) = @_; if ($args{db_type} eq 'SQLite3') { - $ENV{PATH}="@SQLITE_BINDIR@:$ENV{PATH}"; - if ($mode eq 'dump') { - dump_sqlite3(%args); - } + $ENV{PATH}="@SQLITE_BINDIR@:$ENV{PATH}"; + if ($mode eq 'dump') { + dump_sqlite3(%args); + } } elsif ($args{db_type} eq 'PostgreSQL') { - $ENV{PATH}="@POSTGRESQL_BINDIR@:$ENV{PATH}"; - if ($mode eq 'dump') { - dump_pgsql(%args); - } else { - analyse_pgsql(%args); - } + $ENV{PATH}="@POSTGRESQL_BINDIR@:$ENV{PATH}"; + if ($mode eq 'dump') { + dump_pgsql(%args); + } else { + analyse_pgsql(%args); + } } elsif ($args{db_type} eq 'MySQL') { - $ENV{PATH}="@MYSQL_BINDIR@:$ENV{PATH}"; - if ($mode eq 'dump') { - dump_mysql(%args); - } else { - analyse_mysql(%args); - } + $ENV{PATH}="@MYSQL_BINDIR@:$ENV{PATH}"; + if ($mode eq 'dump') { + dump_mysql(%args); + } else { + analyse_mysql(%args); + } } else { - die "This database type isn't supported"; + die "This database type isn't supported"; } } @@ -174,14 +176,14 @@ my %cfg; while(my $l = ) { if ($l =~ /catalog=(.+)/) { - if (exists $cfg{catalog} and $cfg{catalog} eq $cat) { - exit handle_catalog($mode, %cfg); - } - %cfg = (); # reset + if (exists $cfg{catalog} and $cfg{catalog} eq $cat) { + exit handle_catalog($mode, %cfg); + } + %cfg = (); # reset } if ($l =~ /(\w+)=(.+)/) { - $cfg{$1}=$2; + $cfg{$1}=$2; } } diff --git a/bacula/src/cats/make_mysql_tables.in b/bacula/src/cats/make_mysql_tables.in index 2e78e585dc..148cf61021 100644 --- a/bacula/src/cats/make_mysql_tables.in +++ b/bacula/src/cats/make_mysql_tables.in @@ -2,13 +2,12 @@ # # shell script to create Bacula MySQL tables # -# Copyright (C) 2000-2016 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # -# Important note: -# We cannot provide support for performance issues -# if you changed the default schema. In partcular -# if you change any of the indexes. +# Important note: +# You won't get any support for performance issue if you changed the +# default # schema. # # Useful commands: # mysql -u root @@ -54,7 +53,7 @@ CREATE TABLE Path ( -- configuration parameters concerning sorting, joining and global -- memory. By default, sort and join parameters are very small -- (sometimes 8Kb), and having sufficient memory specified by those --- parameters is extremely important to run fast. +-- parameters is extremely important to run fast. -- In File table -- FileIndex can be 0 for FT_DELETED files @@ -164,10 +163,12 @@ CREATE TABLE Job ( Comment BLOB, FileTable CHAR(20) DEFAULT 'File', PRIMARY KEY(JobId), - INDEX (Name(128)) + INDEX (Name(128)), + INDEX (JobTDate) ); --- Create a table like Job for long term statistics + +-- Create a table like Job for long term statistics CREATE TABLE JobHisto ( JobId INTEGER UNSIGNED NOT NULL, Job TINYBLOB NOT NULL, @@ -223,7 +224,7 @@ CREATE TABLE LocationLog ( ); -# +# CREATE TABLE FileSet ( FileSetId INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, FileSet TINYBLOB NOT NULL, @@ -262,13 +263,16 @@ CREATE TABLE Media ( VolJobs INTEGER UNSIGNED DEFAULT 0, VolFiles INTEGER UNSIGNED DEFAULT 0, VolBlocks INTEGER UNSIGNED DEFAULT 0, + VolParts INTEGER UNSIGNED DEFAULT 0, + VolCloudParts INTEGER UNSIGNED DEFAULT 0, VolMounts INTEGER UNSIGNED DEFAULT 0, VolBytes BIGINT UNSIGNED DEFAULT 0, VolABytes BIGINT UNSIGNED DEFAULT 0, VolAPadding BIGINT UNSIGNED DEFAULT 0, VolHoleBytes BIGINT UNSIGNED DEFAULT 0, VolHoles INTEGER UNSIGNED DEFAULT 0, - VolParts INTEGER UNSIGNED DEFAULT 0, /* Now used for VolType */ + LastPartBytes BIGINT UNSIGNED DEFAULT 0, + VolType INTEGER UNSIGNED DEFAULT 0, VolErrors INTEGER UNSIGNED DEFAULT 0, VolWrites BIGINT UNSIGNED DEFAULT 0, VolCapacityBytes BIGINT UNSIGNED DEFAULT 0, @@ -277,6 +281,7 @@ CREATE TABLE Media ( Enabled TINYINT DEFAULT 1, Recycle TINYINT DEFAULT 0, ActionOnPurge TINYINT DEFAULT 0, + CacheRetention BIGINT UNSIGNED DEFAULT 0, VolRetention BIGINT UNSIGNED DEFAULT 0, VolUseDuration BIGINT UNSIGNED DEFAULT 0, MaxVolJobs INTEGER UNSIGNED DEFAULT 0, @@ -310,6 +315,7 @@ CREATE TABLE Pool ( UseOnce TINYINT DEFAULT 0, UseCatalog TINYINT DEFAULT 0, AcceptAnyVolume TINYINT DEFAULT 0, + CacheRetention BIGINT UNSIGNED DEFAULT 0, VolRetention BIGINT UNSIGNED DEFAULT 0, VolUseDuration BIGINT UNSIGNED DEFAULT 0, MaxVolJobs INTEGER UNSIGNED DEFAULT 0, @@ -355,7 +361,7 @@ CREATE TABLE Log ( CREATE TABLE BaseFiles ( - BaseId INTEGER UNSIGNED AUTO_INCREMENT, + BaseId BIGINT UNSIGNED AUTO_INCREMENT, BaseJobId INTEGER UNSIGNED NOT NULL REFERENCES Job, JobId INTEGER UNSIGNED NOT NULL REFERENCES Job, FileId BIGINT UNSIGNED NOT NULL REFERENCES File, @@ -428,7 +434,7 @@ CREATE TABLE PathHierarchy CONSTRAINT pathhierarchy_pkey PRIMARY KEY (PathId) ); -CREATE INDEX pathhierarchy_ppathid +CREATE INDEX pathhierarchy_ppathid ON PathHierarchy (PPathId); CREATE TABLE PathVisibility @@ -459,18 +465,18 @@ CREATE TABLE Snapshot ( primary key (SnapshotId) ); -CREATE UNIQUE INDEX snapshot_idx ON Snapshot (Device(255), +CREATE UNIQUE INDEX snapshot_idx ON Snapshot (Device(255), Volume(255), Name(255)); CREATE TABLE Version ( - VersionId INTEGER UNSIGNED NOT NULL + VersionId INTEGER UNSIGNED NOT NULL ); --- Initialize Version -INSERT INTO Version (VersionId) VALUES (15); +-- Initialize Version +INSERT INTO Version (VersionId) VALUES (16); END-OF-DATA then diff --git a/bacula/src/cats/make_postgresql_tables.in b/bacula/src/cats/make_postgresql_tables.in index 5faba4b972..43833c4a9f 100644 --- a/bacula/src/cats/make_postgresql_tables.in +++ b/bacula/src/cats/make_postgresql_tables.in @@ -2,7 +2,7 @@ # # shell script to create Bacula PostgreSQL tables # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # # Important note: @@ -111,7 +111,7 @@ CREATE TABLE Job RealEndTime timestamp without time zone, JobTDate bigint default 0, VolSessionId integer default 0, - volSessionTime integer default 0, + VolSessionTime integer default 0, JobFiles integer default 0, JobBytes bigint default 0, ReadBytes bigint default 0, @@ -130,6 +130,7 @@ CREATE TABLE Job ); CREATE INDEX job_name_idx on job (name text_pattern_ops); +CREATE INDEX job_jobtdate_idx on job (jobtdate); -- Create a table like Job for long term statistics CREATE TABLE JobHisto (LIKE Job); @@ -188,16 +189,19 @@ CREATE TABLE media voljobs integer default 0, volfiles integer default 0, volblocks integer default 0, + volparts integer default 0, + volcloudparts integer default 0, volmounts integer default 0, volbytes bigint default 0, volabytes bigint default 0, volapadding bigint default 0, volholebytes bigint default 0, volholes integer default 0, - volparts integer default 0, /* Now used for VolType */ + voltype integer default 0, volerrors integer default 0, volwrites bigint default 0, volcapacitybytes bigint default 0, + lastpartbytes bigint default 0, volstatus text not null check (volstatus in ('Full','Archive','Append', 'Recycle','Purged','Read-Only','Disabled', @@ -205,6 +209,7 @@ CREATE TABLE media enabled smallint default 1, recycle smallint default 0, ActionOnPurge smallint default 0, + cacheretention bigint default 0, volretention bigint default 0, voluseduration bigint default 0, maxvoljobs integer default 0, @@ -274,6 +279,7 @@ CREATE TABLE pool useonce smallint default 0, usecatalog smallint default 0, acceptanyvolume smallint default 0, + cacheretention bigint default 0, volretention bigint default 0, voluseduration bigint default 0, maxvoljobs integer default 0, @@ -351,7 +357,7 @@ CREATE TABLE counters CREATE TABLE basefiles ( - baseid serial not null, + baseid bigserial not null, jobid integer not null, fileid bigint not null, fileindex integer , @@ -476,7 +482,7 @@ CREATE UNIQUE INDEX snapshot_idx ON Snapshot (Device text_pattern_ops, Volume text_pattern_ops, Name text_pattern_ops); -INSERT INTO Version (VersionId) VALUES (15); +INSERT INTO Version (VersionId) VALUES (16); -- Make sure we have appropriate permissions diff --git a/bacula/src/cats/make_sqlite3_tables.in b/bacula/src/cats/make_sqlite3_tables.in index 5b4322b4c8..507adaf4b3 100644 --- a/bacula/src/cats/make_sqlite3_tables.in +++ b/bacula/src/cats/make_sqlite3_tables.in @@ -2,7 +2,7 @@ # # shell script to create Bacula SQLite tables # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @@ -102,6 +102,7 @@ CREATE TABLE Job ( PRIMARY KEY(JobId) ); CREATE INDEX inx6 ON Job (Name); +CREATE INDEX job_jobtdate_inx ON Job (JobTDate); -- Create a table like Job for long term statistics CREATE TABLE JobHisto ( @@ -205,13 +206,16 @@ CREATE TABLE Media ( VolJobs INTEGER UNSIGNED DEFAULT 0, VolFiles INTEGER UNSIGNED DEFAULT 0, VolBlocks INTEGER UNSIGNED DEFAULT 0, + LastPartBytes BIGINT UNSIGNED DEFAULT 0, VolMounts INTEGER UNSIGNED DEFAULT 0, VolBytes BIGINT UNSIGNED DEFAULT 0, VolABytes BIGINT UNSIGNED DEFAULT 0, VolAPadding BIGINT UNSIGNED DEFAULT 0, VolHoleBytes BIGINT UNSIGNED DEFAULT 0, VolHoles INTEGER UNSIGNED DEFAULT 0, - VolParts INTEGER UNSIGNED DEFAULT 0, /* Now used for VolType */ + VolType INTEGER UNSIGNED DEFAULT 0, + VolParts INTERGER UNSIGNED DEFAULT 0, + VolCloudParts INTERGER UNSIGNED DEFAULT 0, VolErrors INTEGER UNSIGNED DEFAULT 0, VolWrites BIGINT UNSIGNED DEFAULT 0, VolCapacityBytes BIGINT UNSIGNED DEFAULT 0, @@ -219,6 +223,7 @@ CREATE TABLE Media ( Enabled TINYINT DEFAULT 1, Recycle TINYINT DEFAULT 0, ActionOnPurge TINYINT DEFAULT 0, + CacheRetention BIGINT UNSIGNED DEFAULT 0, VolRetention BIGINT UNSIGNED DEFAULT 0, VolUseDuration BIGINT UNSIGNED DEFAULT 0, MaxVolJobs INTEGER UNSIGNED DEFAULT 0, @@ -287,6 +292,7 @@ CREATE TABLE Pool ( UseOnce TINYINT DEFAULT 0, UseCatalog TINYINT DEFAULT 1, AcceptAnyVolume TINYINT DEFAULT 0, + CacheRetention BIGINT UNSIGNED DEFAULT 0, VolRetention BIGINT UNSIGNED DEFAULT 0, VolUseDuration BIGINT UNSIGNED DEFAULT 0, MaxVolJobs INTEGER UNSIGNED DEFAULT 0, @@ -322,7 +328,7 @@ CREATE TABLE Client ( ); CREATE TABLE BaseFiles ( - BaseId INTEGER, + BaseId BIGINT, BaseJobId INTEGER UNSIGNED REFERENCES Job NOT NULL, JobId INTEGER UNSIGNED REFERENCES Job NOT NULL, FileId INTEGER UNSIGNED REFERENCES File NOT NULL, diff --git a/bacula/src/cats/mysql.c b/bacula/src/cats/mysql.c index 529e6c55c3..cdb75e1f98 100644 --- a/bacula/src/cats/mysql.c +++ b/bacula/src/cats/mysql.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -28,7 +28,6 @@ * change for a C++ programmer, nothing substantial was done, yet all the * code was recommitted under this programmer's name. Consequently, we * undo those changes here. - * */ #include "bacula.h" @@ -37,7 +36,7 @@ #include "cats.h" #include -#define BDB_MYSQL_H_ 1 +#define __BDB_MYSQL_H_ 1 #include "bdb_mysql.h" /* ----------------------------------------------------------------------- @@ -52,7 +51,7 @@ static dlist *db_list = NULL; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -BDB_MYSQL::BDB_MYSQL() +BDB_MYSQL::BDB_MYSQL(): BDB() { BDB_MYSQL *mdb = this; @@ -82,7 +81,7 @@ BDB_MYSQL::BDB_MYSQL() db_list->append(this); } -BDB_MYSQL::~BDB_MYSQL() +BDB_MYSQL::~BDB_MYSQL() { } diff --git a/bacula/src/cats/postgresql.c b/bacula/src/cats/postgresql.c index 6794b4ae74..dabc4b827b 100644 --- a/bacula/src/cats/postgresql.c +++ b/bacula/src/cats/postgresql.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -58,7 +58,7 @@ static dlist *db_list = NULL; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -BDB_POSTGRESQL::BDB_POSTGRESQL() +BDB_POSTGRESQL::BDB_POSTGRESQL(): BDB() { BDB_POSTGRESQL *mdb = this; diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h index e6a9c220a8..e281435ef6 100644 --- a/bacula/src/cats/protos.h +++ b/bacula/src/cats/protos.h @@ -75,11 +75,11 @@ BDB *db_init_database(JCR *jcr, const char *db_driver, const char *db_name, #define db_start_transaction(jcr, mdb) \ mdb->bdb_start_transaction(jcr) #define db_end_transaction(jcr, mdb) \ - mdb->bdb_end_transaction(jcr) + if (mdb) mdb->bdb_end_transaction(jcr) #define db_sql_query(mdb, query, result_handler, ctx) \ mdb->bdb_sql_query(query, result_handler, ctx) #define db_thread_cleanup(mdb) \ - mdb->bdb_thread_cleanup() + if (mdb) mdb->bdb_thread_cleanup() /* sql.c */ int db_int64_handler(void *ctx, int num_fields, char **row); @@ -148,7 +148,8 @@ void bdb_free_restoreobject_record(JCR *jcr, ROBJECT_DBR *rr); bdb_disable_batch_insert(disable) #define db_create_snapshot_record(jcr, mdb, sr) \ mdb->bdb_create_snapshot_record(jcr, sr) - +#define db_get_job_statistics(jcr, mdb, jr) \ + mdb->bdb_get_job_statistics(jcr, jr) /* sql_delete.c */ #define db_delete_pool_record(jcr, mdb, pool_dbr) \ @@ -238,8 +239,8 @@ void bdb_free_restoreobject_record(JCR *jcr, ROBJECT_DBR *rr); mdb->bdb_list_job_records(jcr, jr, sendit, ctx, type) #define db_list_job_totals(jcr, mdb, jr, sendit, ctx) \ mdb->bdb_list_job_totals(jcr, jr, sendit, ctx) -#define db_list_files_for_job(jcr, mdb, jobid, sendit, ctx) \ - mdb->bdb_list_files_for_job(jcr, jobid, sendit, ctx) +#define db_list_files_for_job(jcr, mdb, jobid, deleted, sendit, ctx) \ + mdb->bdb_list_files_for_job(jcr, jobid, deleted, sendit, ctx) #define db_list_media_records(jcr, mdb, mdbr, sendit, ctx, type) \ mdb->bdb_list_media_records(jcr, mdbr, sendit, ctx, type) #define db_list_jobmedia_records(jcr, mdb, JobId, sendit, ctx, type) \ diff --git a/bacula/src/cats/sql.c b/bacula/src/cats/sql.c index eb8aefb860..3b9598d936 100644 --- a/bacula/src/cats/sql.c +++ b/bacula/src/cats/sql.c @@ -1,7 +1,7 @@ -/* +/* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,11 +11,11 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. -*/ +*/ /* * Bacula Catalog Database interface routines * @@ -166,7 +166,191 @@ static int db_max_connections_handler(void *ctx, int num_fields, char **row) } return 0; } - + +BDB::BDB() +{ + init_acl(); + acl_join = get_pool_memory(PM_MESSAGE); + acl_where = get_pool_memory(PM_MESSAGE); +} + +BDB::~BDB() +{ + free_acl(); + free_pool_memory(acl_join); + free_pool_memory(acl_where); +} + +/* Get the WHERE section of a query that permits to respect + * the console ACLs. + * + * get_acls(DB_ACL_BIT(DB_ACL_JOB) | DB_ACL_BIT(DB_ACL_CLIENT), true) + * -> WHERE Job.Name IN ('a', 'b', 'c') AND Client.Name IN ('d', 'e') + * + * get_acls(DB_ACL_BIT(DB_ACL_JOB) | DB_ACL_BIT(DB_ACL_CLIENT), false) + * -> AND Job.Name IN ('a', 'b', 'c') AND Client.Name IN ('d', 'e') + */ +char *BDB::get_acls(int tables, bool where /* use WHERE or AND */) +{ + POOL_MEM tmp; + pm_strcpy(acl_where, ""); + + for (int i=0 ; i < DB_ACL_LAST; i++) { + if (tables & DB_ACL_BIT(i)) { + pm_strcat(acl_where, get_acl((DB_ACL_t)i, where)); + where = acl_where[0] == 0 && where; + } + } + return acl_where; +} + +/* Create the JOIN string that will help to filter queries results */ +char *BDB::get_acl_join_filter(int tables) +{ + POOL_MEM tmp; + pm_strcpy(acl_join, ""); + + if (tables & DB_ACL_BIT(DB_ACL_JOB)) { + Mmsg(tmp, " JOIN Job USING (JobId) "); + pm_strcat(acl_join, tmp); + } + if (tables & (DB_ACL_BIT(DB_ACL_CLIENT) | DB_ACL_BIT(DB_ACL_RCLIENT) | DB_ACL_BIT(DB_ACL_BCLIENT))) { + Mmsg(tmp, " JOIN Client USING (ClientId) "); + pm_strcat(acl_join, tmp); + } + if (tables & DB_ACL_BIT(DB_ACL_POOL)) { + Mmsg(tmp, " JOIN Pool USING (PoolId) "); + pm_strcat(acl_join, tmp); + } + if (tables & DB_ACL_BIT(DB_ACL_PATH)) { + Mmsg(tmp, " JOIN Path USING (PathId) "); + pm_strcat(acl_join, tmp); + } + if (tables & DB_ACL_BIT(DB_ACL_LOG)) { + Mmsg(tmp, " JOIN Log USING (JobId) "); + pm_strcat(acl_join, tmp); + } + if (tables & DB_ACL_BIT(DB_ACL_FILESET)) { + Mmsg(tmp, " LEFT JOIN FileSet USING (FileSetId) "); + pm_strcat(acl_join, tmp); + } + return acl_join; +} + +/* Intialize the ACL list */ +void BDB::init_acl() +{ + for(int i=0; i < DB_ACL_LAST; i++) { + acls[i] = NULL; + } +} + +/* Free ACL list */ +void BDB::free_acl() +{ + for(int i=0; i < DB_ACL_LAST; i++) { + free_and_null_pool_memory(acls[i]); + } +} + +/* Get ACL for a given type */ +const char *BDB::get_acl(DB_ACL_t type, bool where /* display WHERE or AND */) +{ + if (!acls[type]) { + return ""; + } + strcpy(acls[type], where?" WHERE ":" AND "); + acls[type][7] = ' ' ; /* replace \0 by ' ' */ + return acls[type]; +} + +/* Keep UAContext ACLs in our structure for further SQL queries */ +void BDB::set_acl(JCR *jcr, DB_ACL_t type, alist *list, alist *list2) +{ + /* If the list is present, but we authorize everything */ + if (list && list->size() == 1 && strcasecmp((char*)list->get(0), "*all*") == 0) { + return; + } + + /* If the list is present, but we authorize everything */ + if (list2 && list2->size() == 1 && strcasecmp((char*)list2->get(0), "*all*") == 0) { + return; + } + + POOLMEM *tmp = get_pool_memory(PM_FNAME); + POOLMEM *where = get_pool_memory(PM_FNAME); + + *where = 0; + *tmp = 0; + + /* For clients, we can have up to 2 lists */ + escape_acl_list(jcr, &tmp, list); + escape_acl_list(jcr, &tmp, list2); + + switch(type) { + case DB_ACL_JOB: + Mmsg(where, " AND Job.Name IN (%s) ", tmp); + break; + case DB_ACL_CLIENT: + Mmsg(where, " AND Client.Name IN (%s) ", tmp); + break; + case DB_ACL_BCLIENT: + Mmsg(where, " AND Client.Name IN (%s) ", tmp); + break; + case DB_ACL_RCLIENT: + Mmsg(where, " AND Client.Name IN (%s) ", tmp); + break; + case DB_ACL_FILESET: + Mmsg(where, " AND (FileSetId = 0 OR FileSet.FileSet IN (%s)) ", tmp); + break; + case DB_ACL_POOL: + Mmsg(where, " AND (PoolId = 0 OR Pool.Name IN (%s)) ", tmp); + break; + default: + break; + } + acls[type] = where; + free_pool_memory(tmp); +} + +/* Convert a ACL list to a SQL IN() list */ +char *BDB::escape_acl_list(JCR *jcr, POOLMEM **escaped_list, alist *lst) +{ + char *elt; + int len; + POOL_MEM tmp; + + if (!lst) { + return *escaped_list; /* TODO: check how we handle the empty list */ + + /* List is empty, reject everything */ + } else if (lst->size() == 0) { + Mmsg(escaped_list, "''"); + return *escaped_list; + } + + foreach_alist(elt, lst) { + if (elt && *elt) { + len = strlen(elt); + /* Escape + ' ' */ + tmp.check_size(2 * len + 2 + 2); + + pm_strcpy(tmp, "'"); + bdb_lock(); + bdb_escape_string(jcr, tmp.c_str() + 1 , elt, len); + bdb_unlock(); + pm_strcat(tmp, "'"); + + if (*escaped_list[0]) { + pm_strcat(escaped_list, ","); + } + + pm_strcat(escaped_list, tmp.c_str()); + } + } + return *escaped_list; +} + /* * Check catalog max_connections setting */ @@ -795,4 +979,9 @@ void bdb_debug_print(JCR *jcr, FILE *fp) mdb->print_lock_info(fp); } +bool BDB::bdb_check_settings(JCR *jcr, int64_t *starttime, int val, int64_t val2) +{ + return true; +} + #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL */ diff --git a/bacula/src/cats/sql_cmds.c b/bacula/src/cats/sql_cmds.c index ebd439c571..76b5bec85a 100644 --- a/bacula/src/cats/sql_cmds.c +++ b/bacula/src/cats/sql_cmds.c @@ -1001,6 +1001,15 @@ static const char *expired_volumes_defaults = " AND ( Media.LastWritten + Media.VolRetention ) < NOW()" " %s "; +const char *prune_cache[] = { + /* MySQL */ + " (Media.LastWritten + Media.CacheRetention) < NOW() ", + /* PostgreSQL */ + " (Media.LastWritten + (interval '1 second' * Media.CacheRetention)) < NOW() ", + /* SQLite */ + " ( strftime('%s', Media.LastWritten) + Media.CacheRetention < strftime('%s', datetime('now', 'localtime'))) " +}; + const char *expired_volumes[] = { /* MySQL */ expired_volumes_defaults, @@ -1016,9 +1025,18 @@ const char *expired_volumes[] = { const char *expires_in[] = { /* MySQL */ - "(GREATEST(0, CAST(UNIX_TIMESTAMP(LastWritten) + VolRetention AS SIGNED) - UNIX_TIMESTAMP(NOW())))", + "(GREATEST(0, CAST(UNIX_TIMESTAMP(LastWritten) + Media.VolRetention AS SIGNED) - UNIX_TIMESTAMP(NOW())))", + /* PostgreSQL */ + "GREATEST(0, (extract('epoch' from LastWritten + Media.VolRetention * interval '1second' - NOW())::bigint))", + /* SQLite */ + "MAX(0, (strftime('%s', LastWritten) + Media.VolRetention - strftime('%s', datetime('now', 'localtime'))))" +}; + +const char *strip_restore[] = { + /* MySQL */ + "DELETE FROM %s WHERE FileId IN (SELECT * FROM (SELECT FileId FROM %s as B JOIN File USING (FileId) WHERE PathId IN (%s)) AS C)", /* PostgreSQL */ - "GREATEST(0, (extract('epoch' from LastWritten + VolRetention * interval '1second' - NOW())::bigint))", + "DELETE FROM %s WHERE FileId IN (SELECT FileId FROM %s JOIN File USING (FileId) WHERE PathId IN (%s))", /* SQLite */ - "MAX(0, (strftime('%s', LastWritten) + VolRetention - strftime('%s', datetime('now', 'localtime'))))" + "DELETE FROM %s WHERE FileId IN (SELECT FileId FROM %s JOIN File USING (FileId) WHERE PathId IN (%s))" }; diff --git a/bacula/src/cats/sql_cmds.h b/bacula/src/cats/sql_cmds.h index 43ded93f50..85211cbcd9 100644 --- a/bacula/src/cats/sql_cmds.h +++ b/bacula/src/cats/sql_cmds.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -87,3 +87,5 @@ extern const char CATS_IMP_EXP *uar_sel_filesetid; extern const char CATS_IMP_EXP *uar_sel_jobid_temp; extern const char CATS_IMP_EXP *update_counter_values[]; extern const char CATS_IMP_EXP *expires_in[]; +extern const char CATS_IMP_EXP *prune_cache[]; +extern const char CATS_IMP_EXP *strip_restore[]; diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index 255d407652..fbdcf356db 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -21,12 +20,11 @@ * Bacula Catalog Database Create record interface routines * * Written by Kern Sibbald, March 2000 - * */ #include "bacula.h" -static const int dbglevel = 100; +static const int dbglevel = 160; #if HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL @@ -131,7 +129,7 @@ bool BDB::bdb_create_jobmedia_record(JCR *jcr, JOBMEDIA_DBR *jm) } else { /* Worked, now update the Media record with the EndFile and EndBlock */ Mmsg(cmd, - "UPDATE Media SET EndFile=%u, EndBlock=%u WHERE MediaId=%u", + "UPDATE Media SET EndFile=%lu, EndBlock=%lu WHERE MediaId=%lu", jm->EndFile, jm->EndBlock, jm->MediaId); if (!UpdateDB(jcr, cmd, false)) { Mmsg2(&errmsg, _("Update Media record %s failed: ERR=%s\n"), cmd, @@ -151,7 +149,7 @@ bool BDB::bdb_create_jobmedia_record(JCR *jcr, JOBMEDIA_DBR *jm) bool BDB::bdb_create_pool_record(JCR *jcr, POOL_DBR *pr) { bool stat; - char ed1[30], ed2[30], ed3[50], ed4[50], ed5[50]; + char ed1[30], ed2[30], ed3[50], ed4[50], ed5[50], ed6[50]; char esc_name[MAX_ESCAPE_NAME_LENGTH]; char esc_lf[MAX_ESCAPE_NAME_LENGTH]; @@ -178,8 +176,8 @@ bool BDB::bdb_create_pool_record(JCR *jcr, POOL_DBR *pr) "INSERT INTO Pool (Name,NumVols,MaxVols,UseOnce,UseCatalog," "AcceptAnyVolume,AutoPrune,Recycle,VolRetention,VolUseDuration," "MaxVolJobs,MaxVolFiles,MaxVolBytes,PoolType,LabelType,LabelFormat," -"RecyclePoolId,ScratchPoolId,ActionOnPurge) " -"VALUES ('%s',%u,%u,%d,%d,%d,%d,%d,%s,%s,%u,%u,%s,'%s',%d,'%s',%s,%s,%d)", +"RecyclePoolId,ScratchPoolId,ActionOnPurge,CacheRetention) " +"VALUES ('%s',%u,%u,%d,%d,%d,%d,%d,%s,%s,%u,%u,%s,'%s',%d,'%s',%s,%s,%d,%s)", esc_name, pr->NumVols, pr->MaxVols, pr->UseOnce, pr->UseCatalog, @@ -192,7 +190,8 @@ bool BDB::bdb_create_pool_record(JCR *jcr, POOL_DBR *pr) pr->PoolType, pr->LabelType, esc_lf, edit_int64(pr->RecyclePoolId,ed4), edit_int64(pr->ScratchPoolId,ed5), - pr->ActionOnPurge + pr->ActionOnPurge, + edit_uint64(pr->CacheRetention, ed6) ); Dmsg1(200, "Create Pool: %s\n", cmd); if ((pr->PoolId = sql_insert_autokey_record(cmd, NT_("Pool"))) == 0) { @@ -367,7 +366,7 @@ int BDB::bdb_create_media_record(JCR *jcr, MEDIA_DBR *mr) { int stat; char ed1[50], ed2[50], ed3[50], ed4[50], ed5[50], ed6[50], ed7[50], ed8[50]; - char ed9[50], ed10[50], ed11[50], ed12[50]; + char ed9[50], ed10[50], ed11[50], ed12[50], ed13[50], ed14[50]; struct tm tm; char esc_name[MAX_ESCAPE_NAME_LENGTH]; char esc_mtype[MAX_ESCAPE_NAME_LENGTH]; @@ -396,11 +395,12 @@ int BDB::bdb_create_media_record(JCR *jcr, MEDIA_DBR *mr) Mmsg(cmd, "INSERT INTO Media (VolumeName,MediaType,MediaTypeId,PoolId,MaxVolBytes," "VolCapacityBytes,Recycle,VolRetention,VolUseDuration,MaxVolJobs,MaxVolFiles," -"VolStatus,Slot,VolBytes,InChanger,VolReadTime,VolWriteTime,VolParts," +"VolStatus,Slot,VolBytes,InChanger,VolReadTime,VolWriteTime,VolType," +"VolParts,VolCloudParts,LastPartBytes," "EndFile,EndBlock,LabelType,StorageId,DeviceId,LocationId," -"ScratchPoolId,RecyclePoolId,Enabled,ActionOnPurge)" -"VALUES ('%s','%s',0,%u,%s,%s,%d,%s,%s,%u,%u,'%s',%d,%s,%d,%s,%s,%d,0,0,%d,%s," -"%s,%s,%s,%s,%d,%d)", +"ScratchPoolId,RecyclePoolId,Enabled,ActionOnPurge,CacheRetention)" +"VALUES ('%s','%s',0,%u,%s,%s,%d,%s,%s,%u,%u,'%s',%d,%s,%d,%s,%s,%d," + "%d,%d,'%s',%d,%d,%d,%s,%s,%s,%s,%s,%d,%d,%s)", esc_name, esc_mtype, mr->PoolId, edit_uint64(mr->MaxVolBytes,ed1), @@ -416,17 +416,23 @@ int BDB::bdb_create_media_record(JCR *jcr, MEDIA_DBR *mr) mr->InChanger, edit_int64(mr->VolReadTime, ed6), edit_int64(mr->VolWriteTime, ed7), - mr->VolType, /* formerly VolParts */ + mr->VolType, + mr->VolParts, + mr->VolCloudParts, + edit_uint64(mr->LastPartBytes, ed8), + mr->EndFile, + mr->EndBlock, mr->LabelType, - edit_int64(mr->StorageId, ed8), - edit_int64(mr->DeviceId, ed9), - edit_int64(mr->LocationId, ed10), - edit_int64(mr->ScratchPoolId, ed11), - edit_int64(mr->RecyclePoolId, ed12), - mr->Enabled, mr->ActionOnPurge + edit_int64(mr->StorageId, ed9), + edit_int64(mr->DeviceId, ed10), + edit_int64(mr->LocationId, ed11), + edit_int64(mr->ScratchPoolId, ed12), + edit_int64(mr->RecyclePoolId, ed13), + mr->Enabled, + mr->ActionOnPurge, + edit_uint64(mr->CacheRetention, ed14) ); - Dmsg1(500, "Create Volume: %s\n", cmd); if ((mr->MediaId = sql_insert_autokey_record(cmd, NT_("Media"))) == 0) { Mmsg2(&errmsg, _("Create DB Media record %s failed. ERR=%s\n"), @@ -442,7 +448,7 @@ int BDB::bdb_create_media_record(JCR *jcr, MEDIA_DBR *mr) (void)localtime_r(&mr->LabelDate, &tm); strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm); Mmsg(cmd, "UPDATE Media SET LabelDate='%s' " - "WHERE MediaId=%d", dt, mr->MediaId); + "WHERE MediaId=%lu", dt, mr->MediaId); stat = UpdateDB(jcr, cmd, false); } /* @@ -614,7 +620,7 @@ int BDB::bdb_create_counter_record(JCR *jcr, COUNTER_DBR *cr) return 1; } bdb_escape_string(jcr, esc, cr->Counter, strlen(cr->Counter)); - + /* Must create it */ Mmsg(cmd, insert_counter_values[bdb_get_type_index()], esc, cr->MinValue, cr->MaxValue, cr->CurrentValue, @@ -802,23 +808,21 @@ bool bdb_write_batch_file_records(JCR *jcr) goto bail_out; } - /* - * We have to lock tables - */ - if (!db_sql_query(jcr->db_batch, batch_lock_filename_query[db_get_type_index(jcr->db_batch)], NULL, NULL)) { + /* We have to lock tables */ + if (!db_sql_query(jcr->db_batch, batch_lock_filename_query[db_get_type_index(jcr->db_batch)], NULL, NULL)) { Jmsg1(jcr, M_FATAL, 0, "Lock Filename table %s\n", jcr->db_batch->errmsg); - goto bail_out; + goto bail_out; } - if (!db_sql_query(jcr->db_batch, batch_fill_filename_query[db_get_type_index(jcr->db_batch)], NULL, NULL)) { + if (!db_sql_query(jcr->db_batch, batch_fill_filename_query[db_get_type_index(jcr->db_batch)], NULL, NULL)) { Jmsg1(jcr,M_FATAL,0,"Fill Filename table %s\n",jcr->db_batch->errmsg); - db_sql_query(jcr->db_batch, batch_unlock_tables_query[db_get_type_index(jcr->db_batch)], NULL, NULL); - goto bail_out; + db_sql_query(jcr->db_batch, batch_unlock_tables_query[db_get_type_index(jcr->db_batch)], NULL, NULL); + goto bail_out; } - if (!db_sql_query(jcr->db_batch, batch_unlock_tables_query[db_get_type_index(jcr->db_batch)], NULL, NULL)) { + if (!db_sql_query(jcr->db_batch, batch_unlock_tables_query[db_get_type_index(jcr->db_batch)], NULL, NULL)) { Jmsg1(jcr, M_FATAL, 0, "Unlock Filename table %s\n", jcr->db_batch->errmsg); - goto bail_out; + goto bail_out; } if (!db_sql_query(jcr->db_batch, @@ -828,7 +832,7 @@ bool bdb_write_batch_file_records(JCR *jcr) "FROM batch " "JOIN Path ON (batch.Path = Path.Path) " "JOIN Filename ON (batch.Name = Filename.Name)", - NULL, NULL)) + NULL, NULL)) { Jmsg1(jcr, M_FATAL, 0, "Fill File table %s\n", jcr->db_batch->errmsg); goto bail_out; @@ -844,7 +848,7 @@ bail_out: return retval; } -/** +/* * Create File record in BDB * * In order to reduce database size, we store the File attributes, @@ -874,8 +878,8 @@ bool BDB::bdb_create_batch_file_attributes_record(JCR *jcr, ATTR_DBR *ar) return false; /* error already printed */ } if (!jcr->db_batch->sql_batch_start(jcr)) { - Mmsg1(&errmsg, - "Can't start batch mode: ERR=%s", db_strerror(jcr->db_batch)); + Mmsg1(&errmsg, + "Can't start batch mode: ERR=%s", jcr->db_batch->bdb_strerror()); Jmsg(jcr, M_FATAL, 0, "%s", errmsg); return false; } @@ -1003,7 +1007,7 @@ int BDB::bdb_create_filename_record(JCR *jcr, ATTR_DBR *ar) Mmsg(cmd, "INSERT INTO Filename (Name) VALUES ('%s')", esc_name); ar->FilenameId = sql_insert_autokey_record(cmd, NT_("Filename")); - if (ar->FilenameId == 0) { + if (ar->FilenameId == 0) { Mmsg2(&errmsg, _("Create db Filename record %s failed. ERR=%s\n"), cmd, sql_strerror()); Jmsg(jcr, M_FATAL, 0, "%s", errmsg); diff --git a/bacula/src/cats/sql_delete.c b/bacula/src/cats/sql_delete.c index b52c379651..a46a3114a4 100644 --- a/bacula/src/cats/sql_delete.c +++ b/bacula/src/cats/sql_delete.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -20,7 +20,6 @@ * Bacula Catalog Database Delete record interface routines * * Written by Kern Sibbald, December 2000-2014 - * */ #include "bacula.h" @@ -149,7 +148,7 @@ static int do_media_purge(BDB *mdb, MEDIA_DBR *mr) del.tot_ids = 0; del.num_del = 0; del.max_ids = 0; - Mmsg(mdb->cmd, "SELECT JobId from JobMedia WHERE MediaId=%d", mr->MediaId); + Mmsg(mdb->cmd, "SELECT JobId from JobMedia WHERE MediaId=%lu", mr->MediaId); del.max_ids = mr->VolJobs; if (del.max_ids < 100) { del.max_ids = 100; @@ -189,7 +188,7 @@ int BDB::bdb_delete_media_record(JCR *jcr, MEDIA_DBR *mr) do_media_purge(this, mr); } - Mmsg(cmd, "DELETE FROM Media WHERE MediaId=%d", mr->MediaId); + Mmsg(cmd, "DELETE FROM Media WHERE MediaId=%lu", mr->MediaId); bdb_sql_query(cmd, NULL, (void *)NULL); bdb_unlock(); return 1; diff --git a/bacula/src/cats/sql_find.c b/bacula/src/cats/sql_find.c index 8adba6c32b..ae7aeb5039 100644 --- a/bacula/src/cats/sql_find.c +++ b/bacula/src/cats/sql_find.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -24,7 +24,6 @@ * request are in get.c * * Written by Kern Sibbald, December 2000 - * */ #include "bacula.h" @@ -384,11 +383,13 @@ int BDB::bdb_find_next_volume(JCR *jcr, int item, bool InChanger, MEDIA_DBR *mr) "VolBytes,VolMounts,VolErrors,VolWrites,MaxVolBytes,VolCapacityBytes," "MediaType,VolStatus,PoolId,VolRetention,VolUseDuration,MaxVolJobs," "MaxVolFiles,Recycle,Slot,FirstWritten,LastWritten,InChanger," - "EndFile,EndBlock,VolParts,LabelType,LabelDate,StorageId," + "EndFile,EndBlock,VolType,VolParts,VolCloudParts,LastPartBytes," + "LabelType,LabelDate,StorageId," "Enabled,LocationId,RecycleCount,InitialWrite," - "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge " - "FROM Media WHERE PoolId=%s AND MediaType='%s' AND VolStatus IN ('Full'," - "'Recycle','Purged','Used','Append') AND Enabled=1 " + "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge,CacheRetention " + "FROM Media WHERE PoolId=%s AND MediaType='%s' " + " AND (VolStatus = 'Append' OR (VolStatus IN ('Recycle', 'Purged', 'Used', 'Full') AND Recycle=1)) " + " AND Enabled=1 " "ORDER BY LastWritten LIMIT 1", edit_int64(mr->PoolId, ed1), esc_type); item = 1; @@ -397,9 +398,21 @@ int BDB::bdb_find_next_volume(JCR *jcr, int item, bool InChanger, MEDIA_DBR *mr) POOL_MEM voltype(PM_FNAME); POOL_MEM exclude(PM_FNAME); /* Find next available volume */ + /* ***FIXME*** + * replace switch with + * if (StorageId == 0) + * break; + * else use mr->sid_group; but it must be set!!! + */ if (InChanger) { - Mmsg(changer, " AND InChanger=1 AND StorageId=%s ", + ASSERT(mr->sid_group); + if (mr->sid_group) { + Mmsg(changer, " AND InChanger=1 AND StorageId IN (%s) ", + mr->sid_group); + } else { + Mmsg(changer, " AND InChanger=1 AND StorageId=%s ", edit_int64(mr->StorageId, ed1)); + } } /* Volumes will be automatically excluded from the query, we just take the * first one of the list @@ -414,13 +427,19 @@ int BDB::bdb_find_next_volume(JCR *jcr, int item, bool InChanger, MEDIA_DBR *mr) } else { order = sql_media_order_most_recently_written[bdb_get_type_index()]; /* take most recently written */ } + if (mr->VolType == 0) { + Mmsg(voltype, ""); + } else { + Mmsg(voltype, "AND VolType IN (0,%d)", mr->VolType); + } Mmsg(cmd, "SELECT MediaId,VolumeName,VolJobs,VolFiles,VolBlocks," "VolBytes,VolMounts,VolErrors,VolWrites,MaxVolBytes,VolCapacityBytes," "MediaType,VolStatus,PoolId,VolRetention,VolUseDuration,MaxVolJobs," "MaxVolFiles,Recycle,Slot,FirstWritten,LastWritten,InChanger," - "EndFile,EndBlock,VolParts,LabelType,LabelDate,StorageId," + "EndFile,EndBlock,VolType,VolParts,VolCloudParts,LastPartBytes," + "LabelType,LabelDate,StorageId," "Enabled,LocationId,RecycleCount,InitialWrite," - "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge " + "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge,CacheRetention " "FROM Media WHERE PoolId=%s AND MediaType='%s' AND Enabled=1 " "AND VolStatus='%s' " "%s " @@ -491,21 +510,25 @@ int BDB::bdb_find_next_volume(JCR *jcr, int item, bool InChanger, MEDIA_DBR *mr) mr->InChanger = str_to_uint64(row[22]); mr->EndFile = str_to_uint64(row[23]); mr->EndBlock = str_to_uint64(row[24]); - mr->VolType = str_to_int64(row[25]); /* formerly VolParts */ - mr->LabelType = str_to_int64(row[26]); - bstrncpy(mr->cLabelDate, row[27]!=NULL?row[27]:"", sizeof(mr->cLabelDate)); + mr->VolType = str_to_int64(row[25]); + mr->VolParts = str_to_int64(row[26]); + mr->VolCloudParts = str_to_int64(row[27]); + mr->LastPartBytes = str_to_int64(row[28]); + mr->LabelType = str_to_int64(row[29]); + bstrncpy(mr->cLabelDate, row[30]!=NULL?row[30]:"", sizeof(mr->cLabelDate)); mr->LabelDate = (time_t)str_to_utime(mr->cLabelDate); - mr->StorageId = str_to_int64(row[28]); - mr->Enabled = str_to_int64(row[29]); - mr->LocationId = str_to_int64(row[30]); - mr->RecycleCount = str_to_int64(row[31]); - bstrncpy(mr->cInitialWrite, row[32]!=NULL?row[32]:"", sizeof(mr->cInitialWrite)); + mr->StorageId = str_to_int64(row[31]); + mr->Enabled = str_to_int64(row[32]); + mr->LocationId = str_to_int64(row[33]); + mr->RecycleCount = str_to_int64(row[34]); + bstrncpy(mr->cInitialWrite, row[35]!=NULL?row[35]:"", sizeof(mr->cInitialWrite)); mr->InitialWrite = (time_t)str_to_utime(mr->cInitialWrite); - mr->ScratchPoolId = str_to_int64(row[33]); - mr->RecyclePoolId = str_to_int64(row[34]); - mr->VolReadTime = str_to_int64(row[35]); - mr->VolWriteTime = str_to_int64(row[36]); - mr->ActionOnPurge = str_to_int64(row[37]); + mr->ScratchPoolId = str_to_int64(row[36]); + mr->RecyclePoolId = str_to_int64(row[37]); + mr->VolReadTime = str_to_int64(row[38]); + mr->VolWriteTime = str_to_int64(row[39]); + mr->ActionOnPurge = str_to_int64(row[40]); + mr->CacheRetention = str_to_int64(row[41]); sql_free_result(); diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index d6ae982560..d2f26273c8 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -23,7 +23,6 @@ * should be in find.c * * Written by Kern Sibbald, March 2000 - * */ #include "bacula.h" @@ -517,7 +516,7 @@ int BDB::bdb_get_pool_ids(JCR *jcr, int *num_ids, uint32_t *ids[]) bdb_lock(); *ids = NULL; - Mmsg(cmd, "SELECT PoolId FROM Pool"); + Mmsg(cmd, "SELECT PoolId FROM Pool ORDER By Name"); if (QueryDB(jcr, cmd)) { *num_ids = sql_num_rows(); if (*num_ids > 0) { @@ -593,7 +592,7 @@ bool BDB::bdb_get_pool_record(JCR *jcr, POOL_DBR *pdbr) "SELECT PoolId,Name,NumVols,MaxVols,UseOnce,UseCatalog,AcceptAnyVolume," "AutoPrune,Recycle,VolRetention,VolUseDuration,MaxVolJobs,MaxVolFiles," "MaxVolBytes,PoolType,LabelType,LabelFormat,RecyclePoolId,ScratchPoolId," -"ActionOnPurge FROM Pool WHERE Pool.PoolId=%s", +"ActionOnPurge,CacheRetention FROM Pool WHERE Pool.PoolId=%s", edit_int64(pdbr->PoolId, ed1)); } else { /* find by name */ bdb_escape_string(jcr, esc, pdbr->Name, strlen(pdbr->Name)); @@ -601,7 +600,7 @@ bool BDB::bdb_get_pool_record(JCR *jcr, POOL_DBR *pdbr) "SELECT PoolId,Name,NumVols,MaxVols,UseOnce,UseCatalog,AcceptAnyVolume," "AutoPrune,Recycle,VolRetention,VolUseDuration,MaxVolJobs,MaxVolFiles," "MaxVolBytes,PoolType,LabelType,LabelFormat,RecyclePoolId,ScratchPoolId," -"ActionOnPurge FROM Pool WHERE Pool.Name='%s'", esc); +"ActionOnPurge,CacheRetention FROM Pool WHERE Pool.Name='%s'", esc); } if (QueryDB(jcr, cmd)) { if (sql_num_rows() > 1) { @@ -634,6 +633,7 @@ bool BDB::bdb_get_pool_record(JCR *jcr, POOL_DBR *pdbr) pdbr->RecyclePoolId = str_to_int64(row[17]); pdbr->ScratchPoolId = str_to_int64(row[18]); pdbr->ActionOnPurge = str_to_int32(row[19]); + pdbr->CacheRetention = str_to_int64(row[20]); ok = true; } } @@ -986,8 +986,13 @@ bool BDB::bdb_get_media_ids(JCR *jcr, MEDIA_DBR *mr, int *num_ids, uint32_t *ids bdb_lock(); *ids = NULL; - Mmsg(cmd, "SELECT DISTINCT MediaId FROM Media WHERE Recycle=%d AND Enabled=%d ", - mr->Recycle, mr->Enabled); + Mmsg(cmd, "SELECT DISTINCT MediaId FROM Media WHERE Enabled=%d ", + mr->Enabled); + + if (mr->Recycle >= 0) { + bsnprintf(buf, sizeof(buf), "AND Recycle=%d ", mr->Recycle); + pm_strcat(cmd, buf); + } if (*mr->MediaType) { bdb_escape_string(jcr, esc, mr->MediaType, strlen(mr->MediaType)); @@ -995,7 +1000,10 @@ bool BDB::bdb_get_media_ids(JCR *jcr, MEDIA_DBR *mr, int *num_ids, uint32_t *ids pm_strcat(cmd, buf); } - if (mr->StorageId) { + if (mr->sid_group) { + bsnprintf(buf, sizeof(buf), "AND StorageId IN (%s) ", mr->sid_group); + pm_strcat(cmd, buf); + } else if (mr->StorageId) { bsnprintf(buf, sizeof(buf), "AND StorageId=%s ", edit_uint64(mr->StorageId, ed1)); pm_strcat(cmd, buf); } @@ -1022,6 +1030,12 @@ bool BDB::bdb_get_media_ids(JCR *jcr, MEDIA_DBR *mr, int *num_ids, uint32_t *ids pm_strcat(cmd, buf); } + /* Filter the volumes with the CacheRetention */ + if (mr->CacheRetention) { + bsnprintf(buf, sizeof(buf), "AND %s ", prune_cache[bdb_get_type_index()]); + pm_strcat(cmd, buf); + } + Dmsg1(100, "q=%s\n", cmd); if (QueryDB(jcr, cmd)) { @@ -1044,7 +1058,6 @@ bool BDB::bdb_get_media_ids(JCR *jcr, MEDIA_DBR *mr, int *num_ids, uint32_t *ids return ok; } - /** * This function returns a list of all the DBIds that are returned * for the query. @@ -1108,9 +1121,10 @@ bool BDB::bdb_get_media_record(JCR *jcr, MEDIA_DBR *mr) "VolErrors,VolWrites,MaxVolBytes,VolCapacityBytes," "MediaType,VolStatus,PoolId,VolRetention,VolUseDuration,MaxVolJobs," "MaxVolFiles,Recycle,Slot,FirstWritten,LastWritten,InChanger," - "EndFile,EndBlock,VolParts,LabelType,LabelDate,StorageId," + "EndFile,EndBlock,VolType,VolParts,VolCloudParts,LastPartBytes," + "LabelType,LabelDate,StorageId," "Enabled,LocationId,RecycleCount,InitialWrite," - "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge " + "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge,CacheRetention " "FROM Media WHERE MediaId=%s", edit_int64(mr->MediaId, ed1)); } else { /* find by name */ @@ -1120,9 +1134,10 @@ bool BDB::bdb_get_media_record(JCR *jcr, MEDIA_DBR *mr) "VolErrors,VolWrites,MaxVolBytes,VolCapacityBytes," "MediaType,VolStatus,PoolId,VolRetention,VolUseDuration,MaxVolJobs," "MaxVolFiles,Recycle,Slot,FirstWritten,LastWritten,InChanger," - "EndFile,EndBlock,VolParts,LabelType,LabelDate,StorageId," + "EndFile,EndBlock,VolType,VolParts,VolCloudParts,LastPartBytes," + "LabelType,LabelDate,StorageId," "Enabled,LocationId,RecycleCount,InitialWrite," - "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge " + "ScratchPoolId,RecyclePoolId,VolReadTime,VolWriteTime,ActionOnPurge,CacheRetention " "FROM Media WHERE VolumeName='%s'", esc); } @@ -1137,7 +1152,6 @@ bool BDB::bdb_get_media_record(JCR *jcr, MEDIA_DBR *mr) Mmsg1(errmsg, _("error fetching row: %s\n"), sql_strerror()); Jmsg(jcr, M_ERROR, 0, "%s", errmsg); } else { - /* return values */ mr->MediaId = str_to_int64(row[0]); bstrncpy(mr->VolumeName, row[1]!=NULL?row[1]:"", sizeof(mr->VolumeName)); mr->VolJobs = str_to_int64(row[2]); @@ -1168,21 +1182,25 @@ bool BDB::bdb_get_media_record(JCR *jcr, MEDIA_DBR *mr) mr->InChanger = str_to_uint64(row[25]); mr->EndFile = str_to_uint64(row[26]); mr->EndBlock = str_to_uint64(row[27]); - mr->VolType = str_to_int64(row[28]); /* formerly VolParts */ - mr->LabelType = str_to_int64(row[29]); - bstrncpy(mr->cLabelDate, row[30]!=NULL?row[30]:"", sizeof(mr->cLabelDate)); + mr->VolType = str_to_int64(row[28]); + mr->VolParts = str_to_int64(row[29]); + mr->VolCloudParts = str_to_int64(row[30]); + mr->LastPartBytes = str_to_uint64(row[31]); + mr->LabelType = str_to_int64(row[32]); + bstrncpy(mr->cLabelDate, row[33]!=NULL?row[33]:"", sizeof(mr->cLabelDate)); mr->LabelDate = (time_t)str_to_utime(mr->cLabelDate); - mr->StorageId = str_to_int64(row[31]); - mr->Enabled = str_to_int64(row[32]); - mr->LocationId = str_to_int64(row[33]); - mr->RecycleCount = str_to_int64(row[34]); - bstrncpy(mr->cInitialWrite, row[35]!=NULL?row[35]:"", sizeof(mr->cInitialWrite)); + mr->StorageId = str_to_int64(row[34]); + mr->Enabled = str_to_int64(row[35]); + mr->LocationId = str_to_int64(row[36]); + mr->RecycleCount = str_to_int64(row[37]); + bstrncpy(mr->cInitialWrite, row[38]!=NULL?row[38]:"", sizeof(mr->cInitialWrite)); mr->InitialWrite = (time_t)str_to_utime(mr->cInitialWrite); - mr->ScratchPoolId = str_to_int64(row[36]); - mr->RecyclePoolId = str_to_int64(row[37]); - mr->VolReadTime = str_to_int64(row[38]); - mr->VolWriteTime = str_to_int64(row[39]); - mr->ActionOnPurge = str_to_int32(row[40]); + mr->ScratchPoolId = str_to_int64(row[39]); + mr->RecyclePoolId = str_to_int64(row[40]); + mr->VolReadTime = str_to_int64(row[41]); + mr->VolWriteTime = str_to_int64(row[42]); + mr->ActionOnPurge = str_to_int32(row[43]); + mr->CacheRetention = str_to_int64(row[44]); ok = true; } @@ -1288,6 +1306,10 @@ bool BDB::bdb_get_used_base_jobids(JCR *jcr, return bdb_sql_query(buf.c_str(), db_list_handler, result); } +/* Mutex used to have global counter on btemp table */ +static pthread_mutex_t btemp_mutex = PTHREAD_MUTEX_INITIALIZER; +static uint32_t btemp_cur = 1; + /** * The decision do change an incr/diff was done before * Full : do nothing @@ -1313,9 +1335,20 @@ bool BDB::bdb_get_accurate_jobids(JCR *jcr, bstrutime(date, sizeof(date), StartTime + 1); jobids->reset(); + /* If we are comming from bconsole, we must ensure that we + * have a unique name. + */ + if (jcr->JobId == 0) { + P(btemp_mutex); + bsnprintf(jobid, sizeof(jobid), "0%u", btemp_cur++); + V(btemp_mutex); + } else { + edit_uint64(jcr->JobId, jobid); + } + /* First, find the last good Full backup for this job/client/fileset */ Mmsg(query, create_temp_accurate_jobids[bdb_get_type_index()], - edit_uint64(jcr->JobId, jobid), + jobid, edit_uint64(jr->ClientId, clientid), date, edit_uint64(jr->FileSetId, filesetid)); @@ -1552,4 +1585,135 @@ bool BDB::bdb_get_snapshot_record(JCR *jcr, SNAPSHOT_DBR *sr) return ok; } +/* Job, Level */ +static void build_estimate_query(BDB *db, POOL_MEM &query, const char *mode, + char *job_esc, char level) +{ + POOL_MEM filter, tmp; + char ed1[50]; + + + if (level == 0) { + level = 'F'; + } + /* MySQL doesn't have statistic functions */ + if (db->bdb_get_type_index() == SQL_TYPE_POSTGRESQL) { + /* postgresql have functions that permit to handle lineal regression + * in y=ax + b + * REGR_SLOPE(Y,X) = get x + * REGR_INTERCEPT(Y,X) = get b + * and we need y when x=now() + * CORR gives the correlation + * (TODO: display progress bar only if CORR > 0.8) + */ + btime_t now = time(NULL); + Mmsg(query, + "SELECT temp.jobname AS jobname, " + "COALESCE(CORR(value,JobTDate),0) AS corr, " + "(%s*REGR_SLOPE(value,JobTDate) " + " + REGR_INTERCEPT(value,JobTDate)) AS value, " + "AVG(value) AS avg_value, " + " COUNT(1) AS nb ", edit_int64(now, ed1)); + } else { + Mmsg(query, + "SELECT jobname AS jobname, " + "0.1 AS corr, AVG(value) AS value, AVG(value) AS avg_value, " + "COUNT(1) AS nb "); + } + + /* if it's a differential, we need to compare since the last full + * + * F D D D F D D D F I I I I D I I I + * | # # # # | # # + * | # # # # # # | # # + * | # # # # # # # # | # # # # # # # # # + * +----------------- +------------------- + */ + if (level == L_DIFFERENTIAL) { + Mmsg(filter, + " AND Job.StartTime > ( " + " SELECT StartTime " + " FROM Job " + " WHERE Job.Name = '%s' " + " AND Job.Level = 'F' " + " AND Job.JobStatus IN ('T', 'W') " + " ORDER BY Job.StartTime DESC LIMIT 1) ", + job_esc); + } + Mmsg(tmp, + " FROM ( " + " SELECT Job.Name AS jobname, " + " %s AS value, " + " JobTDate AS jobtdate " + " FROM Job INNER JOIN Client USING (ClientId) " + " WHERE Job.Name = '%s' " + " AND Job.Level = '%c' " + " AND Job.JobStatus IN ('T', 'W') " + "%s " + "ORDER BY StartTime DESC " + "LIMIT 4" + ") AS temp GROUP BY temp.jobname", + mode, job_esc, level, filter.c_str() + ); + pm_strcat(query, tmp.c_str()); +} + +bool BDB::bdb_get_job_statistics(JCR *jcr, JOB_DBR *jr) +{ + SQL_ROW row; + POOL_MEM queryB, queryF, query; + char job_esc[MAX_ESCAPE_NAME_LENGTH]; + bool ok = false; + + bdb_lock(); + bdb_escape_string(jcr, job_esc, jr->Name, strlen(jr->Name)); + build_estimate_query(this, queryB, "JobBytes", job_esc, jr->JobLevel); + build_estimate_query(this, queryF, "JobFiles", job_esc, jr->JobLevel); + Mmsg(query, + "SELECT bytes.corr * 100 AS corr_jobbytes, " /* 0 */ + "bytes.value AS jobbytes, " /* 1 */ + "bytes.avg_value AS avg_jobbytes, " /* 2 */ + "bytes.nb AS nb_jobbytes, " /* 3 */ + "files.corr * 100 AS corr_jobfiles, " /* 4 */ + "files.value AS jobfiles, " /* 5 */ + "files.avg_value AS avg_jobfiles, " /* 6 */ + "files.nb AS nb_jobfiles " /* 7 */ + "FROM (%s) AS bytes LEFT JOIN (%s) AS files USING (jobname)", + queryB.c_str(), queryF.c_str()); + Dmsg1(100, "query=%s\n", query.c_str()); + + if (QueryDB(jcr, query.c_str())) { + char ed1[50]; + if (sql_num_rows() > 1) { + Mmsg1(errmsg, _("More than one Result!: %s\n"), + edit_uint64(sql_num_rows(), ed1)); + goto bail_out; + } + ok = true; + + if ((row = sql_fetch_row()) == NULL) { + Mmsg1(errmsg, _("error fetching row: %s\n"), sql_strerror()); + } else { + jr->CorrJobBytes = str_to_int64(row[0]); + jr->JobBytes = str_to_int64(row[1]); + + /* lineal expression with only one job doesn't return a correct value */ + if (str_to_int64(row[3]) == 1) { + jr->JobBytes = str_to_int64(row[2]); /* Take the AVG value */ + } + jr->CorrNbJob = str_to_int64(row[3]); /* Number of jobs used in this sample */ + jr->CorrJobFiles = str_to_int64(row[4]); + jr->JobFiles = str_to_int64(row[5]); + + if (str_to_int64(row[7]) == 1) { + jr->JobFiles = str_to_int64(row[6]); /* Take the AVG value */ + } + } + sql_free_result(); + } +bail_out: + bdb_unlock(); + return ok; +} + #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL */ diff --git a/bacula/src/cats/sql_list.c b/bacula/src/cats/sql_list.c index 5fb94f1cf7..1bd7e65a85 100644 --- a/bacula/src/cats/sql_list.c +++ b/bacula/src/cats/sql_list.c @@ -81,13 +81,13 @@ void BDB::bdb_list_pool_records(JCR *jcr, POOL_DBR *pdbr, Mmsg(cmd, "SELECT PoolId,Name,NumVols,MaxVols,UseOnce,UseCatalog," "AcceptAnyVolume,VolRetention,VolUseDuration,MaxVolJobs,MaxVolBytes," "AutoPrune,Recycle,PoolType,LabelFormat,Enabled,ScratchPoolId," - "RecyclePoolId,LabelType " + "RecyclePoolId,LabelType,ActionOnPurge,CacheRetention " " FROM Pool WHERE Name='%s'", esc); } else { Mmsg(cmd, "SELECT PoolId,Name,NumVols,MaxVols,UseOnce,UseCatalog," "AcceptAnyVolume,VolRetention,VolUseDuration,MaxVolJobs,MaxVolBytes," "AutoPrune,Recycle,PoolType,LabelFormat,Enabled,ScratchPoolId," - "RecyclePoolId,LabelType " + "RecyclePoolId,LabelType,ActionOnPurge,CacheRetention " " FROM Pool ORDER BY PoolId"); } } else { @@ -199,47 +199,71 @@ void BDB::bdb_list_media_records(JCR *jcr, MEDIA_DBR *mdbr, bdb_lock(); bdb_escape_string(jcr, esc, mdbr->VolumeName, strlen(mdbr->VolumeName)); + const char *join = ""; + const char *where = ""; if (type == VERT_LIST) { if (mdbr->VolumeName[0] != 0) { Mmsg(cmd, "SELECT MediaId,VolumeName,Slot,PoolId," "MediaType,MediaTypeId,FirstWritten,LastWritten,LabelDate,VolJobs," - "VolFiles,VolBlocks,VolMounts,VolBytes,VolABytes,VolAPadding," - "VolHoleBytes,VolHoles,VolErrors,VolWrites," - "VolCapacityBytes,VolStatus,Enabled,Recycle,VolRetention," - "VolUseDuration,MaxVolJobs,MaxVolFiles,MaxVolBytes,InChanger," - "EndFile,EndBlock,VolParts,LabelType,StorageId,DeviceId," + "VolFiles,VolBlocks,VolParts,VolCloudParts,Media.CacheRetention,VolMounts,VolBytes," + "VolABytes,VolAPadding," + "VolHoleBytes,VolHoles,LastPartBytes,VolErrors,VolWrites," + "VolCapacityBytes,VolStatus,Media.Enabled,Media.Recycle,Media.VolRetention," + "Media.VolUseDuration,Media.MaxVolJobs,Media.MaxVolFiles,Media.MaxVolBytes,InChanger," + "EndFile,EndBlock,VolType,Media.LabelType,StorageId,DeviceId," "MediaAddressing,VolReadTime,VolWriteTime," - "LocationId,RecycleCount,InitialWrite,ScratchPoolId,RecyclePoolId, " - "ActionOnPurge,%s AS ExpiresIn, Comment" - " FROM Media WHERE Media.VolumeName='%s'", expiresin, esc); + "LocationId,RecycleCount,InitialWrite,Media.ScratchPoolId,Media.RecyclePoolId, " + "Media.ActionOnPurge,%s AS ExpiresIn, Comment" + " FROM Media %s WHERE Media.VolumeName='%s' %s", + expiresin, + join, + esc, + where + ); } else { Mmsg(cmd, "SELECT MediaId,VolumeName,Slot,PoolId," "MediaType,MediaTypeId,FirstWritten,LastWritten,LabelDate,VolJobs," - "VolFiles,VolBlocks,VolMounts,VolBytes,VolABytes,VolAPadding," - "VolHoleBytes,VolHoles,VolErrors,VolWrites," - "VolCapacityBytes,VolStatus,Enabled,Recycle,VolRetention," - "VolUseDuration,MaxVolJobs,MaxVolFiles,MaxVolBytes,InChanger," - "EndFile,EndBlock,VolParts,LabelType,StorageId,DeviceId," + "VolFiles,VolBlocks,VolParts,VolCloudParts,Media.CacheRetention,VolMounts,VolBytes," + "VolABytes,VolAPadding," + "VolHoleBytes,VolHoles,LastPartBytes,VolErrors,VolWrites," + "VolCapacityBytes,VolStatus,Media.Enabled,Media.Recycle,Media.VolRetention," + "Media.VolUseDuration,Media.MaxVolJobs,Media.MaxVolFiles,Media.MaxVolBytes,InChanger," + "EndFile,EndBlock,VolType,Media.LabelType,StorageId,DeviceId," "MediaAddressing,VolReadTime,VolWriteTime," - "LocationId,RecycleCount,InitialWrite,ScratchPoolId,RecyclePoolId, " - "ActionOnPurge,%s AS ExpiresIn, Comment" - " FROM Media WHERE Media.PoolId=%s ORDER BY MediaId", - expiresin, edit_int64(mdbr->PoolId, ed1)); + "LocationId,RecycleCount,InitialWrite,Media.ScratchPoolId,Media.RecyclePoolId, " + "Media.ActionOnPurge,%s AS ExpiresIn, Comment" + " FROM Media %s WHERE Media.PoolId=%s %s ORDER BY MediaId", + expiresin, + join, + edit_int64(mdbr->PoolId, ed1), + where + ); } } else { if (mdbr->VolumeName[0] != 0) { - Mmsg(cmd, "SELECT MediaId,VolumeName,VolStatus,Enabled," - "VolBytes,VolFiles,VolRetention,Recycle,Slot,InChanger,MediaType,LastWritten,%s AS ExpiresIn " - "FROM Media WHERE Media.VolumeName='%s'", expiresin, esc); + Mmsg(cmd, "SELECT MediaId,VolumeName,VolStatus,Media.Enabled," + "VolBytes,VolFiles,Media.VolRetention,Media.Recycle,Slot,InChanger,MediaType,VolType," + "VolParts,%s AS ExpiresIn " + "FROM Media %s WHERE Media.VolumeName='%s' %s", + expiresin, + join, + esc, + where + ); } else { - Mmsg(cmd, "SELECT MediaId,VolumeName,VolStatus,Enabled," - "VolBytes,VolFiles,VolRetention,Recycle,Slot,InChanger,MediaType,LastWritten,%s AS ExpiresIn " - "FROM Media WHERE Media.PoolId=%s ORDER BY MediaId", - expiresin, edit_int64(mdbr->PoolId, ed1)); + Mmsg(cmd, "SELECT MediaId,VolumeName,VolStatus,Media.Enabled," + "VolBytes,VolFiles,Media.VolRetention,Media.Recycle,Slot,InChanger,MediaType,VolType," + "VolParts,LastWritten,%s AS ExpiresIn " + "FROM Media %s WHERE Media.PoolId=%s %s ORDER BY MediaId", + expiresin, + join, + edit_int64(mdbr->PoolId, ed1), + where + ); } } - + Dmsg1(DT_SQL|50, "q=%s\n", cmd); if (!QueryDB(jcr, cmd)) { bdb_unlock(); return; @@ -257,30 +281,45 @@ void BDB::bdb_list_jobmedia_records(JCR *jcr, uint32_t JobId, char ed1[50]; bdb_lock(); + const char *join = ""; + const char *where = ""; + if (type == VERT_LIST) { if (JobId > 0) { /* do by JobId */ Mmsg(cmd, "SELECT JobMediaId,JobId,Media.MediaId,Media.VolumeName," "FirstIndex,LastIndex,StartFile,JobMedia.EndFile,StartBlock," "JobMedia.EndBlock " - "FROM JobMedia,Media WHERE Media.MediaId=JobMedia.MediaId " - "AND JobMedia.JobId=%s", edit_int64(JobId, ed1)); + "FROM JobMedia JOIN Media USING (MediaId) %s " + "WHERE JobMedia.JobId=%s %s", + join, + edit_int64(JobId, ed1), + where); } else { Mmsg(cmd, "SELECT JobMediaId,JobId,Media.MediaId,Media.VolumeName," "FirstIndex,LastIndex,StartFile,JobMedia.EndFile,StartBlock," "JobMedia.EndBlock " - "FROM JobMedia,Media WHERE Media.MediaId=JobMedia.MediaId"); + "FROM JobMedia JOIN Media USING (MediaId) %s %s", + join, + where); } } else { if (JobId > 0) { /* do by JobId */ Mmsg(cmd, "SELECT JobId,Media.VolumeName,FirstIndex,LastIndex " - "FROM JobMedia,Media WHERE Media.MediaId=JobMedia.MediaId " - "AND JobMedia.JobId=%s", edit_int64(JobId, ed1)); + "FROM JobMedia JOIN Media USING (MediaId) %s WHERE " + "JobMedia.JobId=%s %s", + join, + edit_int64(JobId, ed1), + where); } else { Mmsg(cmd, "SELECT JobId,Media.VolumeName,FirstIndex,LastIndex " - "FROM JobMedia,Media WHERE Media.MediaId=JobMedia.MediaId"); + "FROM JobMedia JOIN Media USING (MediaId) %s %s", + join, + where); } } + Dmsg1(DT_SQL|50, "q=%s\n", cmd); + if (!QueryDB(jcr, cmd)) { bdb_unlock(); return; @@ -440,7 +479,7 @@ alist *BDB::bdb_list_job_records(JCR *jcr, JOB_DBR *jr, DB_LIST_HANDLER *sendit, "Job.ClientId,Client.Name as ClientName,JobStatus,SchedTime," "StartTime,EndTime,RealEndTime,JobTDate," "VolSessionId,VolSessionTime,JobFiles,JobBytes,ReadBytes,JobErrors," - "JobMissingFiles,Job.PoolId,Pool.Name as PooLname,PriorJobId," + "JobMissingFiles,Job.PoolId,Pool.Name as PoolName,PriorJobId," "Job.FileSetId,FileSet.FileSet,Job.HasBase,Job.HasCache,Job.Comment " "FROM Job JOIN Client USING (ClientId) LEFT JOIN Pool USING (PoolId) " "LEFT JOIN FileSet USING (FileSetId) %s " @@ -521,11 +560,27 @@ void BDB::bdb_list_job_totals(JCR *jcr, JOB_DBR *jr, DB_LIST_HANDLER *sendit, vo bdb_unlock(); } -void BDB::bdb_list_files_for_job(JCR *jcr, JobId_t jobid, DB_LIST_HANDLER *sendit, void *ctx) +/* List all file records from a job + * "deleted" values are described just below + */ +void BDB::bdb_list_files_for_job(JCR *jcr, JobId_t jobid, int deleted, DB_LIST_HANDLER *sendit, void *ctx) { char ed1[50]; + const char *opt; LIST_CTX lctx(jcr, this, sendit, ctx, HORZ_LIST); + switch (deleted) { + case 0: /* Show only actual files */ + opt = " AND FileIndex <> 0 "; + break; + case 1: /* Show only deleted files */ + opt = " AND FileIndex = 0 "; + break; + default: /* Show everything */ + opt = ""; + break; + } + bdb_lock(); /* @@ -533,7 +588,7 @@ void BDB::bdb_list_files_for_job(JCR *jcr, JobId_t jobid, DB_LIST_HANDLER *sendi */ if (bdb_get_type_index() == SQL_TYPE_MYSQL) { Mmsg(cmd, "SELECT CONCAT(Path.Path,Filename.Name) AS Filename " - "FROM (SELECT PathId, FilenameId FROM File WHERE JobId=%s " + "FROM (SELECT PathId, FilenameId FROM File WHERE JobId=%s %s " "UNION ALL " "SELECT PathId, FilenameId " "FROM BaseFiles JOIN File " @@ -542,10 +597,10 @@ void BDB::bdb_list_files_for_job(JCR *jcr, JobId_t jobid, DB_LIST_HANDLER *sendi ") AS F, Filename,Path " "WHERE Filename.FilenameId=F.FilenameId " "AND Path.PathId=F.PathId", - edit_int64(jobid, ed1), ed1); + edit_int64(jobid, ed1), opt, ed1); } else { Mmsg(cmd, "SELECT Path.Path||Filename.Name AS Filename " - "FROM (SELECT PathId, FilenameId FROM File WHERE JobId=%s " + "FROM (SELECT PathId, FilenameId FROM File WHERE JobId=%s %s " "UNION ALL " "SELECT PathId, FilenameId " "FROM BaseFiles JOIN File " @@ -554,9 +609,9 @@ void BDB::bdb_list_files_for_job(JCR *jcr, JobId_t jobid, DB_LIST_HANDLER *sendi ") AS F, Filename,Path " "WHERE Filename.FilenameId=F.FilenameId " "AND Path.PathId=F.PathId", - edit_int64(jobid, ed1), ed1); + edit_int64(jobid, ed1), opt, ed1); } - + Dmsg1(100, "q=%s\n", cmd); if (!bdb_big_sql_query(cmd, list_result, &lctx)) { bdb_unlock(); return; diff --git a/bacula/src/cats/sql_update.c b/bacula/src/cats/sql_update.c index b6c4677c56..ee85bd7e8d 100644 --- a/bacula/src/cats/sql_update.c +++ b/bacula/src/cats/sql_update.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Bacula Catalog Database Update record interface routines * * Written by Kern Sibbald, March 2000 - * */ #include "bacula.h" @@ -58,7 +57,7 @@ int BDB::bdb_add_digest_to_file_record(JCR *jcr, FileId_t FileId, char *digest, bdb_escape_string(jcr, esc_name, digest, len); Mmsg(cmd, "UPDATE File SET MD5='%s' WHERE FileId=%s", esc_name, edit_int64(FileId, ed1)); - ret = UpdateDB(jcr, cmd, true); + ret = UpdateDB(jcr, cmd, false); bdb_unlock(); return ret; } @@ -254,7 +253,7 @@ int BDB::bdb_update_counter_record(JCR *jcr, COUNTER_DBR *cr) int BDB::bdb_update_pool_record(JCR *jcr, POOL_DBR *pr) { int stat; - char ed1[50], ed2[50], ed3[50], ed4[50], ed5[50], ed6[50]; + char ed1[50], ed2[50], ed3[50], ed4[50], ed5[50], ed6[50], ed7[50]; char esc[MAX_ESCAPE_NAME_LENGTH]; bdb_lock(); @@ -270,7 +269,7 @@ int BDB::bdb_update_pool_record(JCR *jcr, POOL_DBR *pr) "AcceptAnyVolume=%d,VolRetention='%s',VolUseDuration='%s'," "MaxVolJobs=%u,MaxVolFiles=%u,MaxVolBytes=%s,Recycle=%d," "AutoPrune=%d,LabelType=%d,LabelFormat='%s',RecyclePoolId=%s," -"ScratchPoolId=%s,ActionOnPurge=%d WHERE PoolId=%s", +"ScratchPoolId=%s,ActionOnPurge=%d,CacheRetention='%s' WHERE PoolId=%s", pr->NumVols, pr->MaxVols, pr->UseOnce, pr->UseCatalog, pr->AcceptAnyVolume, edit_uint64(pr->VolRetention, ed1), edit_uint64(pr->VolUseDuration, ed2), @@ -280,6 +279,7 @@ int BDB::bdb_update_pool_record(JCR *jcr, POOL_DBR *pr) esc, edit_int64(pr->RecyclePoolId,ed5), edit_int64(pr->ScratchPoolId,ed6), pr->ActionOnPurge, + edit_uint64(pr->CacheRetention, ed7), ed4); stat = UpdateDB(jcr, cmd, false); bdb_unlock(); @@ -316,7 +316,7 @@ int BDB::bdb_update_media_record(JCR *jcr, MEDIA_DBR *mr) char ed1[50], ed2[50], ed3[50], ed4[50]; char ed5[50], ed6[50], ed7[50], ed8[50]; char ed9[50], ed10[50], ed11[50], ed12[50]; - char ed13[50], ed14[50]; + char ed13[50], ed14[50], ed15[50], ed16[50]; char esc_name[MAX_ESCAPE_NAME_LENGTH]; char esc_status[MAX_ESCAPE_NAME_LENGTH]; @@ -370,11 +370,12 @@ int BDB::bdb_update_media_record(JCR *jcr, MEDIA_DBR *mr) "VolFiles=%u,VolBlocks=%u,VolBytes=%s,VolABytes=%s," "VolHoleBytes=%s,VolHoles=%u,VolMounts=%u,VolErrors=%u," "VolWrites=%s,MaxVolBytes=%s,VolStatus='%s'," - "Slot=%d,InChanger=%d,VolReadTime=%s,VolWriteTime=%s,VolParts=%d," + "Slot=%d,InChanger=%d,VolReadTime=%s,VolWriteTime=%s,VolType=%d," + "VolParts=%d,VolCloudParts=%d,LastPartBytes=%s," "LabelType=%d,StorageId=%s,PoolId=%s,VolRetention=%s,VolUseDuration=%s," "MaxVolJobs=%d,MaxVolFiles=%d,Enabled=%d,LocationId=%s," "ScratchPoolId=%s,RecyclePoolId=%s,RecycleCount=%d,Recycle=%d," - "ActionOnPurge=%d" + "ActionOnPurge=%d,CacheRetention=%s" " WHERE VolumeName='%s'", mr->VolJobs, mr->VolFiles, mr->VolBlocks, edit_uint64(mr->VolBytes, ed1), @@ -386,17 +387,21 @@ int BDB::bdb_update_media_record(JCR *jcr, MEDIA_DBR *mr) esc_status, mr->Slot, mr->InChanger, edit_int64(mr->VolReadTime, ed6), edit_int64(mr->VolWriteTime, ed7), - mr->VolType, /* formerly VolParts */ + mr->VolType, + mr->VolParts, + mr->VolCloudParts, + edit_uint64(mr->LastPartBytes, ed8), mr->LabelType, - edit_int64(mr->StorageId, ed8), - edit_int64(mr->PoolId, ed9), - edit_uint64(mr->VolRetention, ed10), - edit_uint64(mr->VolUseDuration, ed11), + edit_int64(mr->StorageId, ed9), + edit_int64(mr->PoolId, ed10), + edit_uint64(mr->VolRetention, ed11), + edit_uint64(mr->VolUseDuration, ed12), mr->MaxVolJobs, mr->MaxVolFiles, - mr->Enabled, edit_uint64(mr->LocationId, ed12), - edit_uint64(mr->ScratchPoolId, ed13), - edit_uint64(mr->RecyclePoolId, ed14), + mr->Enabled, edit_uint64(mr->LocationId, ed13), + edit_uint64(mr->ScratchPoolId, ed14), + edit_uint64(mr->RecyclePoolId, ed15), mr->RecycleCount,mr->Recycle, mr->ActionOnPurge, + edit_uint64(mr->CacheRetention, ed16), esc_name); Dmsg1(dbglevel1, "%s\n", cmd); @@ -419,7 +424,7 @@ int BDB::bdb_update_media_record(JCR *jcr, MEDIA_DBR *mr) int BDB::bdb_update_media_defaults(JCR *jcr, MEDIA_DBR *mr) { int stat; - char ed1[50], ed2[50], ed3[50], ed4[50], ed5[50]; + char ed1[50], ed2[50], ed3[50], ed4[50], ed5[50], ed6[50]; char esc[MAX_ESCAPE_NAME_LENGTH]; bool can_be_empty; @@ -428,27 +433,29 @@ int BDB::bdb_update_media_defaults(JCR *jcr, MEDIA_DBR *mr) bdb_escape_string(jcr, esc, mr->VolumeName, strlen(mr->VolumeName)); Mmsg(cmd, "UPDATE Media SET " "ActionOnPurge=%d, Recycle=%d,VolRetention=%s,VolUseDuration=%s," - "MaxVolJobs=%u,MaxVolFiles=%u,MaxVolBytes=%s,RecyclePoolId=%s" + "MaxVolJobs=%u,MaxVolFiles=%u,MaxVolBytes=%s,RecyclePoolId=%s,CacheRetention=%s" " WHERE VolumeName='%s'", mr->ActionOnPurge, mr->Recycle,edit_uint64(mr->VolRetention, ed1), edit_uint64(mr->VolUseDuration, ed2), mr->MaxVolJobs, mr->MaxVolFiles, edit_uint64(mr->MaxVolBytes, ed3), edit_uint64(mr->RecyclePoolId, ed4), + edit_uint64(mr->CacheRetention, ed5), esc); can_be_empty = false; } else { Mmsg(cmd, "UPDATE Media SET " "ActionOnPurge=%d, Recycle=%d,VolRetention=%s,VolUseDuration=%s," - "MaxVolJobs=%u,MaxVolFiles=%u,MaxVolBytes=%s,RecyclePoolId=%s" + "MaxVolJobs=%u,MaxVolFiles=%u,MaxVolBytes=%s,RecyclePoolId=%s,CacheRetention=%s" " WHERE PoolId=%s", mr->ActionOnPurge, mr->Recycle,edit_uint64(mr->VolRetention, ed1), edit_uint64(mr->VolUseDuration, ed2), mr->MaxVolJobs, mr->MaxVolFiles, edit_uint64(mr->MaxVolBytes, ed3), edit_int64(mr->RecyclePoolId, ed4), - edit_int64(mr->PoolId, ed5)); + edit_uint64(mr->CacheRetention, ed5), + edit_int64(mr->PoolId, ed6)); can_be_empty = true; } @@ -469,28 +476,29 @@ int BDB::bdb_update_media_defaults(JCR *jcr, MEDIA_DBR *mr) */ void BDB::bdb_make_inchanger_unique(JCR *jcr, MEDIA_DBR *mr) { - char ed1[50], ed2[50]; + char ed1[50]; char esc[MAX_ESCAPE_NAME_LENGTH]; if (mr->InChanger != 0 && mr->Slot != 0 && mr->StorageId != 0) { + if (!mr->sid_group) { + mr->sid_group = edit_int64(mr->StorageId, mr->sid); + } if (mr->MediaId != 0) { Mmsg(cmd, "UPDATE Media SET InChanger=0, Slot=0 WHERE " - "Slot=%d AND StorageId=%s AND MediaId!=%s", + "Slot=%d AND StorageId IN (%s) AND MediaId!=%s", mr->Slot, - edit_int64(mr->StorageId, ed1), edit_int64(mr->MediaId, ed2)); + mr->sid_group, edit_int64(mr->MediaId, ed1)); } else if (*mr->VolumeName) { bdb_escape_string(jcr, esc,mr->VolumeName,strlen(mr->VolumeName)); Mmsg(cmd, "UPDATE Media SET InChanger=0, Slot=0 WHERE " - "Slot=%d AND StorageId=%s AND VolumeName!='%s'", - mr->Slot, - edit_int64(mr->StorageId, ed1), esc); + "Slot=%d AND StorageId IN (%s) AND VolumeName!='%s'", + mr->Slot, mr->sid_group, esc); } else { /* used by ua_label to reset all volume with this slot */ Mmsg(cmd, "UPDATE Media SET InChanger=0, Slot=0 WHERE " - "Slot=%d AND StorageId=%s", - mr->Slot, - edit_int64(mr->StorageId, ed1), mr->VolumeName); + "Slot=%d AND StorageId IN (%s)", + mr->Slot, mr->sid_group, mr->VolumeName); } Dmsg1(dbglevel1, "%s\n", cmd); UpdateDB(jcr, cmd, true); diff --git a/bacula/src/cats/update_bacula_tables.in b/bacula/src/cats/update_bacula_tables.in index 1249077f96..cafeaa178f 100755 --- a/bacula/src/cats/update_bacula_tables.in +++ b/bacula/src/cats/update_bacula_tables.in @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # # This routine alters the appropriately configured diff --git a/bacula/src/cats/update_mysql_tables.in b/bacula/src/cats/update_mysql_tables.in index 08df0d88b3..f33356feda 100644 --- a/bacula/src/cats/update_mysql_tables.in +++ b/bacula/src/cats/update_mysql_tables.in @@ -1,13 +1,13 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # # Shell script to update MySQL tables from Bacula Community version -# 5.0.x, 5.2.x, 7.0.x +# 5.0.x, 5.2.x, 7.0.x, 7.2.x, 7.4.x # echo " " -echo "This script will update a Bacula MySQL database from version 12-14 to 15" +echo "This script will update a Bacula MySQL database from version 12-15 to 16" echo " " echo "Depending on the current version of your catalog," echo "you may have to run this script multiple times." @@ -143,4 +143,48 @@ END-OF-DATA fi fi +if [ "$DBVERSION" -eq 15 ] ; then + if mysql $* -f <msg); - if (strncmp(dir->msg, oldOKhello, sizeof(oldOKhello)-1) != 0) { - sendit(_("Director rejected Hello command\n")); - goto bail_out; - } else { + if (strncmp(dir->msg, oldOKhello, sizeof(oldOKhello)-1) == 0) { /* If Dir version exists, get it */ sscanf(dir->msg, newOKhello, &dir_version); sendit(dir->msg); + /* Check for hello from FD */ + } else if (sscanf(dir->msg, FDOKhello, &fd_version) == 1) { + sendit(dir->msg); + } else { + sendit(_("Director rejected Hello command\n")); + goto bail_out; + } + /* Turn on compression for newer Directors */ + if (dir_version >= 103 && (!cons || cons->comm_compression)) { + dir->set_compress(); + } else { + dir->clear_compress(); } + /* ***FIXME*** should turn on compression for FD if possible */ stop_bsock_timer(tid); return 1; diff --git a/bacula/src/console/bbconsjson.c b/bacula/src/console/bbconsjson.c new file mode 100644 index 0000000000..a40308f51f --- /dev/null +++ b/bacula/src/console/bbconsjson.c @@ -0,0 +1,594 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2016 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * Bacula Console .conf to Json program. + * + * Kern Sibbald, September MMXII + * + */ + +#include "bacula.h" +#include "lib/breg.h" +#include "console_conf.h" +#include "jcr.h" + +/* Imported variables */ +#if defined(_MSC_VER) +extern "C" { // work around visual compiler mangling variables + extern URES res_all; +} +#else +extern URES res_all; +#endif +extern s_kw msg_types[]; +extern RES_TABLE resources[]; + +/* Exported functions */ +void senditf(const char *fmt, ...); +void sendit(const char *buf); + +/* Imported functions */ +extern bool parse_cons_config(CONFIG *config, const char *configfile, int exit_code); + +typedef struct +{ + /* default { { "Director": { "Name": aa, ...} }, { "Job": {..} */ + bool do_list; /* [ {}, {}, ..] or { "aa": {}, "bb": {}, ...} */ + bool do_one; /* { "Name": "aa", "Description": "test, ... } */ + bool do_only_data; /* [ {}, {}, {}, ] */ + char *resource_type; + char *resource_name; + regex_t directive_reg; +} display_filter; + +/* Forward referenced functions */ +static void terminate_console(int sig); +static int check_resources(); +//static void ressendit(void *ua, const char *fmt, ...); +//static void dump_resource_types(); +//static void dump_directives(); +static void dump_json(display_filter *filter); + +/* Static variables */ +static char *configfile = NULL; +static FILE *output = stdout; +static bool teeout = false; /* output to output and stdout */ +static int numdir; +static POOLMEM *args; +static CONFIG *config; + + +#define CONFIG_FILE "bconsole.conf" /* default configuration file */ + +static void usage() +{ + fprintf(stderr, _( +PROG_COPYRIGHT +"\n%sVersion: " VERSION " (" BDATE ") %s %s %s\n\n" +"Usage: bconsjson [options] [config_file]\n" +" -r get resource type \n" +" -n get resource \n" +" -l get only directives matching dirs (use with -r)\n" +" -D get only data\n" +" -c set configuration file to file\n" +" -d set debug level to \n" +" -dt print timestamp in debug output\n" +" -t test - read configuration and exit\n" +" -v verbose\n" +" -? print this message.\n" +"\n"), 2012, "", HOST_OS, DISTNAME, DISTVER); + + exit(1); +} + +/********************************************************************* + * + * Bacula console conf to Json + * + */ +int main(int argc, char *argv[]) +{ + int ch; + bool test_config = false; + display_filter filter; + memset(&filter, 0, sizeof(filter)); + int rtn = 0; + + setlocale(LC_ALL, ""); + bindtextdomain("bacula", LOCALEDIR); + textdomain("bacula"); + + init_stack_dump(); + lmgr_init_thread(); + my_name_is(argc, argv, "bconsole"); + init_msg(NULL, NULL); + working_directory = "/tmp"; + args = get_pool_memory(PM_FNAME); + + while ((ch = getopt(argc, argv, "n:vDabc:d:jl:r:t?")) != -1) { + switch (ch) { + case 'D': + filter.do_only_data = true; + break; + case 'a': +// list_all = true; + break; + + case 'c': /* configuration file */ + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(optarg); + break; + + break; + + case 'd': + if (*optarg == 't') { + dbg_timestamp = true; + } else { + debug_level = atoi(optarg); + if (debug_level <= 0) { + debug_level = 1; + } + } + + case 'l': + filter.do_list = true; + if (regcomp(&filter.directive_reg, optarg, REG_EXTENDED) != 0) { + Jmsg((JCR *)NULL, M_ERROR_TERM, 0, + _("Please use valid -l argument: %s\n"), optarg); + } + break; + + case 'r': + filter.resource_type = optarg; + break; + + case 'n': + filter.resource_name = optarg; + break; + + case 't': + test_config = true; + break; + + case 'v': /* verbose */ + verbose++; + break; + + case '?': + default: + usage(); + exit(1); + } + } + argc -= optind; + argv += optind; + + OSDependentInit(); + + if (argc) { + usage(); + exit(1); + } + + if (filter.do_list && !filter.resource_type) { + usage(); + } + + if (filter.resource_type && filter.resource_name) { + filter.do_one = true; + } + + if (configfile == NULL || configfile[0] == 0) { + configfile = bstrdup(CONFIG_FILE); + } + + if (test_config && verbose > 0) { + char buf[1024]; + find_config_file(configfile, buf, sizeof(buf)); + printf("config_file=%s\n", buf); + } + + config = New(CONFIG()); + config->encode_password(false); + parse_cons_config(config, configfile, M_ERROR_TERM); + + if (!check_resources()) { + Emsg1(M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); + } + + if (test_config) { + terminate_console(0); + exit(0); + } + + dump_json(&filter); + + terminate_console(0); + return rtn; +} + +/* Cleanup and then exit */ +static void terminate_console(int sig) +{ + static bool already_here = false; + + if (already_here) { /* avoid recursive temination problems */ + exit(1); + } + already_here = true; + stop_watchdog(); + delete config; + config = NULL; + free(res_head); + res_head = NULL; + free_pool_memory(args); + lmgr_cleanup_main(); + + if (sig != 0) { + exit(1); + } + return; +} + + +/* + * Dump out all resources in json format. + * Note!!!! This routine must be in this file rather + * than in src/lib/parser_conf.c otherwise the pointers + * will be all messed up. + */ +static void dump_json(display_filter *filter) +{ + int resinx, item; + int directives; + bool first_res; + bool first_directive; + RES_ITEM *items; + RES *res; + HPKT hpkt; + regmatch_t pmatch[32]; + + init_hpkt(hpkt); + + /* List resources and directives */ + if (filter->do_only_data) { + printf("["); + + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } + * or print a single item + */ + } else if (filter->do_one || filter->do_list) { + printf("{"); + + } else { + /* [ { "Client": { "Name": "aa",.. } }, { "Director": { "Name": "bb", ... } } ]*/ + printf("["); + } + + first_res = true; + /* Loop over all resource types */ + for (resinx=0; resources[resinx].name; resinx++) { + /* Skip this resource type */ + if (filter->resource_type && + strcasecmp(filter->resource_type, resources[resinx].name) != 0) { + continue; + } + + directives = 0; + /* Loop over all resources of this type */ + foreach_rblist(res, res_head[resinx]->res_list) { + hpkt.res = res; + items = resources[resinx].items; + if (!items) { + break; + } + + /* Copy the resource into res_all */ + memcpy(&res_all, res, sizeof(res_all)); + + if (filter->resource_name) { + bool skip=true; + /* The Name should be at the first place, so this is not a real loop */ + for (item=0; items[item].name; item++) { + if (strcasecmp(items[item].name, "Name") == 0) { + if (strcasecmp(*(items[item].value), filter->resource_name) == 0) { + skip = false; + } + break; + } + } + if (skip) { /* The name doesn't match, so skip it */ + continue; + } + } + + if (first_res) { + printf("\n"); + + } else { + printf(",\n"); + } + + if (filter->do_only_data) { + printf(" {"); + + } else if (filter->do_one) { + /* Nothing to print */ + + /* When sending the list, the form is: + * { aa: { Name: aa, Description: aadesc...}, bb: { Name: bb + */ + } else if (filter->do_list) { + /* Search and display Name, should be the first item */ + for (item=0; items[item].name; item++) { + if (strcmp(items[item].name, "Name") == 0) { + printf("%s: {\n", quote_string(hpkt.edbuf, *items[item].value)); + break; + } + } + } else { + /* Begin new resource */ + printf("{\n \"%s\": {", resources[resinx].name); + } + + first_res = false; + first_directive = true; + directives = 0; + + for (item=0; items[item].name; item++) { + /* Check user argument -l */ + if (filter->do_list && + regexec(&filter->directive_reg, + items[item].name, 32, pmatch, 0) != 0) + { + continue; + } + + hpkt.ritem = &items[item]; + if (bit_is_set(item, res_all.hdr.item_present)) { + if (!first_directive) printf(","); + if (display_global_item(hpkt)) { + /* Fall-through wanted */ + } else { + printf("\n \"%s\": null", items[item].name); + } + directives++; + first_directive = false; + } + if (items[item].flags & ITEM_LAST) { + display_last(hpkt); /* If last bit set always call to cleanup */ + } + } + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } */ + if (filter->do_only_data || filter->do_list) { + printf("\n }"); /* Finish the Resource with a single } */ + + } else { + if (filter->do_one) { + /* don't print anything */ + } else if (directives) { + printf("\n }\n}"); /* end of resource */ + } else { + printf("}\n}"); + } + } + } /* End loop over all resources of this type */ + } /* End loop all resource types */ + + if (filter->do_only_data) { + printf("\n]\n"); + + } else if (filter->do_one || filter->do_list) { + printf("\n}\n"); + + } else { + printf("\n]\n"); + } + term_hpkt(hpkt); +} + + +/* + * Make a quick check to see that we have all the + * resources needed. + */ +static int check_resources() +{ + bool OK = true; + DIRRES *director; + bool tls_needed; + + LockRes(); + + numdir = 0; + foreach_res(director, R_DIRECTOR) { + + numdir++; + /* tls_require implies tls_enable */ + if (director->tls_require) { + if (have_tls) { + director->tls_enable = true; + } else { + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; + } + } + + tls_needed = director->tls_enable || director->tls_authenticate; + + if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && tls_needed) { + Emsg2(M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Director \"%s\" in %s." + " At least one CA certificate store is required.\n"), + director->hdr.name, configfile); + OK = false; + } + } + + if (numdir == 0) { + Emsg1(M_FATAL, 0, _("No Director resource defined in %s\n" + "Without that I don't how to speak to the Director :-(\n"), configfile); + OK = false; + } + + CONRES *cons; + /* Loop over Consoles */ + foreach_res(cons, R_CONSOLE) { + /* tls_require implies tls_enable */ + if (cons->tls_require) { + if (have_tls) { + cons->tls_enable = true; + } else { + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; + } + } + tls_needed = cons->tls_enable || cons->tls_authenticate; + if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) && tls_needed) { + Emsg2(M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Console \"%s\" in %s.\n"), + cons->hdr.name, configfile); + OK = false; + } + } + + UnlockRes(); + + return OK; +} + +#ifdef needed +static void ressendit(void *sock, const char *fmt, ...) +{ + char buf[3000]; + va_list arg_ptr; + + va_start(arg_ptr, fmt); + bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); + va_end(arg_ptr); + sendit(buf); +} + +static void dump_resource_types() +{ + int i; + bool first; + + /* List resources and their code */ + printf("[\n"); + first = true; + for (i=0; resources[i].name; i++) { + if (!first) { + printf(",\n"); + } + printf(" \"%s\": %d", resources[i].name, resources[i].rcode); + first = false; + } + printf("\n]\n"); +} + +static void dump_directives() +{ + int i, j; + bool first_res; + bool first_directive; + RES_ITEM *items; + + /* List resources and directives */ + printf("[\n"); + first_res = true; + for (i=0; resources[i].name; i++) { + if (!first_res) { + printf(",\n"); + } + printf("{\n \"%s\": {\n", resources[i].name); + first_res = false; + first_directive = true; + items = resources[i].items; + for (j=0; items[j].name; j++) { + if (!first_directive) { + printf(",\n"); + } + printf(" \"%s\": null", items[j].name); + first_directive = false; + } + printf("\n }"); /* end of resource */ + } + printf("\n]\n"); +} +#endif + +/* + * Send a line to the output file and or the terminal + */ +void senditf(const char *fmt,...) +{ + char buf[3000]; + va_list arg_ptr; + + va_start(arg_ptr, fmt); + bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); + va_end(arg_ptr); + sendit(buf); +} + +void sendit(const char *buf) +{ +#ifdef CONIO_FIX + char obuf[3000]; + if (output == stdout || teeout) { + const char *p, *q; + /* + * Here, we convert every \n into \r\n because the + * terminal is in raw mode when we are using + * conio. + */ + for (p=q=buf; (p=strchr(q, '\n')); ) { + int len = p - q; + if (len > 0) { + memcpy(obuf, q, len); + } + memcpy(obuf+len, "\r\n", 3); + q = ++p; /* point after \n */ + fputs(obuf, output); + } + if (*q) { + fputs(q, output); + } + fflush(output); + } + if (output != stdout) { + fputs(buf, output); + } +#else + + fputs(buf, output); + fflush(output); + if (teeout) { + fputs(buf, stdout); + fflush(stdout); + } +#endif +} diff --git a/bacula/src/console/conio.c b/bacula/src/console/conio.c index 04edb8243a..8a5e117e8b 100755 --- a/bacula/src/console/conio.c +++ b/bacula/src/console/conio.c @@ -1,7 +1,6 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -89,11 +88,6 @@ extern char *BC; extern char *UP; #endif -/* Forward referenced functions */ -extern "C" { -static void sigintcatcher(int); -} - static void add_smap(char *str, int func); @@ -257,9 +251,7 @@ static void asinsl(); static void asdell(); int input_line(char *string, int length); -extern "C" { void con_term(); -} void trapctlc(); int usrbrk(); void clrbrk(); @@ -974,7 +966,7 @@ static void rawmode(FILE *input) /* Defaults, the main program can override these */ signal(SIGQUIT, SIG_IGN); signal(SIGHUP, SIG_IGN); - signal(SIGINT, sigintcatcher); + trapctlc(); signal(SIGWINCH, SIG_IGN); if (!termtype) { @@ -1098,41 +1090,6 @@ t_char(char c) (void)write(1, &c, 1); } - -static int brkflg = 0; /* set on user break */ - -/* Routine to return true if user types break */ -int usrbrk() -{ - return brkflg; -} - -/* Clear break flag */ -void clrbrk() -{ - brkflg = 0; - -} - -/* Interrupt caught here */ -static void sigintcatcher(int sig) -{ - brkflg++; - if (brkflg > 3) { - normode(); - exit(1); - } - signal(SIGINT, sigintcatcher); -} - - -/* Trap Ctl-C */ -void trapctlc() -{ - signal(SIGINT, sigintcatcher); -} - - /* ASCLRL() -- Clear to end of line from current position */ static void asclrl(int pos, int width) { diff --git a/bacula/src/console/conio.h b/bacula/src/console/conio.h index afec01694d..7646ffacac 100644 --- a/bacula/src/console/conio.h +++ b/bacula/src/console/conio.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -22,15 +22,9 @@ extern int input_line(char *line, int len); extern void con_init(FILE *input); -extern "C" { extern void con_term(); -} - extern void con_set_zed_keys(); extern void t_sendl(char *buf, int len); extern void t_send(char *buf); extern void t_char(char c); -extern int usrbrk(void); -extern void clrbrk(void); -extern void trapctlc(void); #endif diff --git a/bacula/src/console/console.c b/bacula/src/console/console.c index 40a1dd4382..01bccf8efd 100644 --- a/bacula/src/console/console.c +++ b/bacula/src/console/console.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -29,18 +29,20 @@ #include "jcr.h" -#ifdef HAVE_CONIO +#if defined(HAVE_CONIO) #include "conio.h" //#define CONIO_FIX 1 -#else +#else /* defined(HAVE_READLINE) || "DUMB" */ #define con_init(x) #define con_term() #define con_set_zed_keys(); -#define trapctlc() -#define clrbrk() -#define usrbrk() 0 #endif +void trapctlc(); +void clrbrk(); +int usrbrk(); +static int brkflg = 0; /* set on user break */ + #if defined(HAVE_WIN32) #define isatty(fd) (fd==0) #endif @@ -51,7 +53,6 @@ /* Imported functions */ int authenticate_director(BSOCK *dir, DIRRES *director, CONRES *cons); -extern bool parse_cons_config(CONFIG *config, const char *configfile, int exit_code); /* Forward referenced functions */ static void terminate_console(int sig); @@ -98,6 +99,7 @@ static int echocmd(FILE *input, BSOCK *UA_sock); static int timecmd(FILE *input, BSOCK *UA_sock); static int sleepcmd(FILE *input, BSOCK *UA_sock); static int execcmd(FILE *input, BSOCK *UA_sock); +static int putfilecmd(FILE *input, BSOCK *UA_sock); #ifdef HAVE_READLINE static int eolcmd(FILE *input, BSOCK *UA_sock); @@ -121,6 +123,8 @@ PROG_COPYRIGHT "Usage: bconsole [-s] [-c config_file] [-d debug_level]\n" " -D select a Director\n" " -l list Directors defined\n" +" -L list Consoles defined\n" +" -C select a console\n" " -c set configuration file to file\n" " -d set debug level to \n" " -dt print timestamp in debug output\n" @@ -180,6 +184,7 @@ static struct cmdstruct commands[] = { { N_("echo"), echocmd, _("echo command string")}, { N_("exec"), execcmd, _("execute an external command")}, { N_("exit"), quitcmd, _("exit = quit")}, + { N_("putfile"), putfilecmd, _("send a file to the director")}, { N_("zed_keys"), zed_keyscmd, _("zed_keys = use zed keys instead of bash keys")}, { N_("help"), helpcmd, _("help listing")}, #ifdef HAVE_READLINE @@ -224,6 +229,39 @@ static int do_a_command(FILE *input, BSOCK *UA_sock) return stat; } +/* When getting .api command, we can ignore some signals, so we set + * api_mode=true + */ +static bool api_mode=false; + +static bool ignore_signal(int stat, BSOCK *s) +{ + /* Not in API mode */ + if (!api_mode) { + return false; + } + + /* not a signal */ + if (stat != -1) { + return false; + } + + /* List signal that should not stop the read loop */ + Dmsg1(100, "Got signal %s\n", bnet_sig_to_ascii(s->msglen)); + switch(s->msglen) { + case BNET_CMD_BEGIN: + case BNET_CMD_FAILED: /* might want to print **ERROR** */ + case BNET_CMD_OK: /* might want to print **OK** */ + case BNET_MSGS_PENDING: + return true; + default: + break; + } + + /* The signal should break the read loop */ + return false; +} + static void read_and_process_input(FILE *input, BSOCK *UA_sock) { const char *prompt = "*"; @@ -241,7 +279,7 @@ static void read_and_process_input(FILE *input, BSOCK *UA_sock) } if (tty_input) { stat = get_cmd(input, prompt, UA_sock, 30); - if (usrbrk() == 1) { + if (usrbrk() >= 1) { clrbrk(); } if (usrbrk()) { @@ -288,12 +326,18 @@ static void read_and_process_input(FILE *input, BSOCK *UA_sock) } stop_bsock_timer(tid); } + if (strncasecmp(UA_sock->msg, ".api", 4) == 0) { + api_mode = true; + } if (strcasecmp(UA_sock->msg, ".quit") == 0 || strcasecmp(UA_sock->msg, ".exit") == 0) { break; } tid = start_bsock_timer(UA_sock, timeout); while (1) { stat = UA_sock->recv(); + if (ignore_signal(stat, UA_sock)) { + continue; + } if (stat < 0) { break; @@ -775,15 +819,8 @@ wait_for_data(int fd, int sec) #if defined(HAVE_WIN32) return 1; #else - fd_set fdset; - struct timeval tv; - - tv.tv_sec = sec; - tv.tv_usec = 0; for ( ;; ) { - FD_ZERO(&fdset); - FD_SET((unsigned)fd, &fdset); - switch(select(fd + 1, &fdset, NULL, NULL, &tv)) { + switch(fd_wait_data(fd, WAIT_READ, sec, 0)) { case 0: /* timeout */ return 0; case -1: @@ -863,6 +900,33 @@ again: #endif /* ! HAVE_READLINE */ +/* Routine to return true if user types break */ +int usrbrk() +{ + return brkflg; +} + +/* Clear break flag */ +void clrbrk() +{ + brkflg = 0; +} + +/* Interrupt caught here */ +static void sigintcatcher(int sig) +{ + brkflg++; + if (brkflg > 3) { + terminate_console(sig); + } + signal(SIGINT, sigintcatcher); +} + +/* Trap Ctl-C */ +void trapctlc() +{ + signal(SIGINT, sigintcatcher); +} static int console_update_history(const char *histfile) { @@ -902,7 +966,8 @@ static int console_init_history(const char *histfile) return ret; } -static bool select_director(const char *director, DIRRES **ret_dir, CONRES **ret_cons) +static bool select_director(const char *director, const char *console, + DIRRES **ret_dir, CONRES **ret_cons) { int numcon=0, numdir=0; int i=0, item=0; @@ -981,7 +1046,11 @@ try_again: /* Look for a console linked to this director */ for (i=0; idirector && strcasecmp(cons->director, dir->hdr.name) == 0) { + if (console) { + if (strcmp(cons->hdr.name, console) == 0) { + break; + } + } else if (cons->director && strcasecmp(cons->director, dir->hdr.name) == 0) { break; } if (i == (numcon - 1)) { @@ -989,6 +1058,12 @@ try_again: } } + if (cons == NULL && console != NULL) { + UnlockRes(); + senditf(_("Can't find %s in Console list\n"), console); + return 0; + } + /* Look for the first non-linked console */ if (cons == NULL) { for (i=0; ihdr.name); + } + UnlockRes(); + } + if (test_config) { terminate_console(0); exit(0); @@ -1157,7 +1253,7 @@ int main(int argc, char *argv[]) start_watchdog(); /* Start socket watchdog */ - if (!select_director(director, &dir, &cons)) { + if (!select_director(director, console, &dir, &cons)) { terminate_console(0); return 1; } @@ -1281,16 +1377,19 @@ static void terminate_console(int sig) } already_here = true; stop_watchdog(); - config->free_resources(); - free(config); + delete(config); config = NULL; cleanup_crypto(); + free(res_head); + res_head = NULL; free_pool_memory(args); +#if defined(HAVE_CONIO) if (!no_conio) { con_term(); } -#ifdef HAVE_READLINE - rl_restore_state(NULL); +#elif defined(HAVE_READLINE) + rl_cleanup_after_signal(); +#else /* !HAVE_CONIO && !HAVE_READLINE */ #endif (void)WSACleanup(); /* Cleanup Windows sockets */ lmgr_cleanup_main(); @@ -1473,15 +1572,30 @@ static int execcmd(FILE *input, BSOCK *UA_sock) char line[5000]; int stat; int wait = 0; + char *cmd; if (argc > 3) { sendit(_("Too many arguments. Enclose command in double quotes.\n")); return 1; } + + /* old syntax */ if (argc == 3) { wait = atoi(argk[2]); } - bpipe = open_bpipe(argk[1], wait, "r"); + cmd = argk[1]; + + /* handle cmd=XXXX and wait=XXXX */ + for (int i=1; ifsend(".putfile key=\"%s\"", key); + + /* Just read the file and send it to the director */ + while (!feof(fp)) { + i = fread(UA_sock->msg, 1, sizeof_pool_memory(UA_sock->msg) - 1, fp); + if (i > 0) { + UA_sock->msg[i] = 0; + UA_sock->msglen = i; + UA_sock->send(); + } + } + + UA_sock->signal(BNET_EOD); + fclose(fp); + + /* Get the file name associated */ + while (UA_sock->recv() > 0) { + senditf("%s", UA_sock->msg); + } + return 1; +} + /* @time */ static int timecmd(FILE *input, BSOCK *UA_sock) { diff --git a/bacula/src/console/console_conf.c b/bacula/src/console/console_conf.c index 8eee61926c..2f647af6d6 100644 --- a/bacula/src/console/console_conf.c +++ b/bacula/src/console/console_conf.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -47,8 +47,7 @@ */ int32_t r_first = R_FIRST; int32_t r_last = R_LAST; -static RES *sres_head[R_LAST - R_FIRST + 1]; -RES **res_head = sres_head; +RES_HEAD **res_head; /* Forward referenced subroutines */ @@ -88,6 +87,7 @@ static RES_ITEM cons_items[] = { {"TlsKey", store_dir, ITEM(res_cons.tls_keyfile), 0, 0, 0}, {"Director", store_str, ITEM(res_cons.director), 0, 0, 0}, {"HeartbeatInterval", store_time, ITEM(res_cons.heartbeat_interval), 0, ITEM_DEFAULT, 5 * 60}, + {"CommCompression", store_bool, ITEM(res_cons.comm_compression), 0, ITEM_DEFAULT, true}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -123,9 +123,9 @@ RES_TABLE resources[] = { }; /* Dump contents of resource */ -void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), void *sock) +void dump_resource(int type, RES *rres, void sendit(void *sock, const char *fmt, ...), void *sock) { - URES *res = (URES *)reshdr; + URES *res = (URES *)rres; bool recurse = true; if (res == NULL) { @@ -138,18 +138,20 @@ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fm } switch (type) { case R_CONSOLE: - printf(_("Console: name=%s rcfile=%s histfile=%s\n"), reshdr->name, + printf(_("Console: name=%s rcfile=%s histfile=%s\n"), rres->name, res->res_cons.rc_file, res->res_cons.hist_file); break; case R_DIRECTOR: - printf(_("Director: name=%s address=%s DIRport=%d\n"), reshdr->name, + printf(_("Director: name=%s address=%s DIRport=%d\n"), rres->name, res->res_dir.address, res->res_dir.DIRport); break; default: printf(_("Unknown resource type %d\n"), type); } - if (recurse && res->res_dir.hdr.next) { - dump_resource(type, res->res_dir.hdr.next, sendit, sock); + + rres = GetNextRes(type, rres); + if (recurse && rres) { + dump_resource(type, rres, sendit, sock); } } @@ -160,17 +162,15 @@ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fm * resource chain is traversed. Mainly we worry about freeing * allocated strings (names). */ -void free_resource(RES *sres, int type) +void free_resource(RES *rres, int type) { - RES *nres; - URES *res = (URES *)sres; + URES *res = (URES *)rres; if (res == NULL) { return; } /* common stuff -- free the resource name */ - nres = (RES *)res->res_dir.hdr.next; if (res->res_dir.hdr.name) { free(res->res_dir.hdr.name); } @@ -204,6 +204,9 @@ void free_resource(RES *sres, int type) if (res->res_cons.director) { free(res->res_cons.director); } + if (res->res_cons.password) { + free(res->res_cons.password); + } break; case R_DIRECTOR: if (res->res_dir.address) { @@ -224,24 +227,23 @@ void free_resource(RES *sres, int type) if (res->res_dir.tls_keyfile) { free(res->res_dir.tls_keyfile); } + if (res->res_dir.password) { + free(res->res_dir.password); + } break; default: printf(_("Unknown resource type %d\n"), type); } /* Common stuff again -- free the resource, recurse to next one */ free(res); - if (nres) { - free_resource(nres, type); - } } /* Save the new resource by chaining it into the head list for * the resource. If this is pass 2, we update any resource * pointers (currently only in the Job resource). */ -void save_resource(int type, RES_ITEM *items, int pass) +bool save_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { - URES *res; int rindex = type - r_first; int i, size; int error = 0; @@ -251,10 +253,11 @@ void save_resource(int type, RES_ITEM *items, int pass) */ for (i=0; items[i].name; i++) { if (items[i].flags & ITEM_REQUIRED) { - if (!bit_is_set(i, res_all.res_dir.hdr.item_present)) { - Emsg2(M_ERROR_TERM, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + if (!bit_is_set(i, res_all.res_dir.hdr.item_present)) { + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), items[i].name, resources[rindex].name); - } + return false; + } } } @@ -286,7 +289,7 @@ void save_resource(int type, RES_ITEM *items, int pass) free(res_all.res_dir.hdr.desc); res_all.res_dir.hdr.desc = NULL; } - return; + return true; } /* The following code is only executed during pass 1 */ @@ -305,30 +308,16 @@ void save_resource(int type, RES_ITEM *items, int pass) } /* Common */ if (!error) { - res = (URES *)malloc(size); - memcpy(res, &res_all, size); - if (!res_head[rindex]) { - res_head[rindex] = (RES *)res; /* store first entry */ - } else { - RES *next, *last; - for (last=next=res_head[rindex]; next; next=next->next) { - last = next; - if (strcmp(next->name, res->res_dir.hdr.name) == 0) { - Emsg2(M_ERROR_TERM, 0, - _("Attempt to define second \"\"%s\" resource named \"%s\" is not permitted.\n"), - resources[rindex].name, res->res_dir.hdr.name); - } - } - last->next = (RES *)res; - Dmsg2(90, "Inserting %s res: %s\n", res_to_str(type), - res->res_dir.hdr.name); + if (!config->insert_res(rindex, size)) { + return false; } } + return true; } bool parse_cons_config(CONFIG *config, const char *configfile, int exit_code) { config->init(configfile, NULL, exit_code, (void *)&res_all, res_all_size, - r_first, r_last, resources, res_head); + r_first, r_last, resources, &res_head); return config->parse_config(); } diff --git a/bacula/src/console/console_conf.h b/bacula/src/console/console_conf.h index 4d5eee7452..ed3ad0f598 100644 --- a/bacula/src/console/console_conf.h +++ b/bacula/src/console/console_conf.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -16,6 +16,11 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ +/* + * Bacula User Agent specific configuration and defines + * + * Kern Sibbald, Sep MM + */ /* * Bacula User Agent specific configuration and defines * @@ -27,6 +32,8 @@ * Resource codes -- they must be sequential for indexing */ +bool parse_cons_config(CONFIG *config, const char *configfile, int exit_code); + enum { R_CONSOLE = 1001, R_DIRECTOR, @@ -54,6 +61,7 @@ struct CONRES { char *rc_file; /* startup file */ char *hist_file; /* command history file */ char *password; /* UA server password */ + bool comm_compression; /* Enable comm line compression */ bool tls_authenticate; /* Authenticate with TLS */ bool tls_enable; /* Enable TLS on all connections */ bool tls_require; /* Require TLS on all connections */ diff --git a/bacula/src/dird/Makefile.in b/bacula/src/dird/Makefile.in index fd727ecd0c..f7478962b3 100644 --- a/bacula/src/dird/Makefile.in +++ b/bacula/src/dird/Makefile.in @@ -1,7 +1,7 @@ # # Bacula Director Makefile # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @MCOMMON@ @@ -26,6 +26,7 @@ CAP_LIBS = @CAP_LIBS@ ZLIBS=@ZLIBS@ DB_LIBS=@DB_LIBS@ + first_rule: all dummy: @@ -46,6 +47,8 @@ SVRSRCS = dird.c admin.c authenticate.c \ ua_status.c ua_tree.c ua_update.c vbackup.c verify.c SVROBJS = $(SVRSRCS:.c=.o) +JSONOBJS = bdirjson.o dird_conf.o run_conf.o inc_conf.o ua_acl.o + # these are the objects that are changed by the .configure process EXTRAOBJS = @OBJLIST@ @@ -58,7 +61,7 @@ EXTRAOBJS = @OBJLIST@ @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< #------------------------------------------------------------------------- -all: Makefile bacula-dir @STATIC_DIR@ +all: Makefile bacula-dir @STATIC_DIR@ bdirjson @echo "==== Make of dird is good ====" @echo " " @@ -78,6 +81,12 @@ static-bacula-dir: Makefile $(SVROBJS) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../ $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) strip $@ +bdirjson: Makefile $(JSONOBJS) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) + @echo "Linking $@ ..." + $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -o $@ $(JSONOBJS) \ + -lbaccfg -lbac -lm $(DLIB) $(DB_LIBS) $(LIBS) \ + $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) + Makefile: $(srcdir)/Makefile.in $(topdir)/config.status cd $(topdir) \ && CONFIG_FILES=$(thisdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status @@ -87,7 +96,7 @@ libtool-clean: clean: libtool-clean @$(RMF) dird bacula-dir core core.* a.out *.o *.bak *~ *.intpro *.extpro 1 2 3 - @$(RMF) static-bacula-dir + @$(RMF) static-bacula-dir bdirjson realclean: clean @$(RMF) tags bacula-dir.conf @@ -102,6 +111,7 @@ devclean: realclean install: all $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-dir $(DESTDIR)$(sbindir)/bacula-dir + $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bdirjson $(DESTDIR)$(sbindir)/bdirjson @srcconf=bacula-dir.conf; \ if test -f ${DESTDIR}${sysconfdir}/$$srcconf; then \ destconf=$$srcconf.new; \ @@ -125,6 +135,7 @@ install: all uninstall: + (cd $(DESTDIR)$(sbindir); $(RMF) bacula-dir bdirjson) (cd $(DESTDIR)$(sysconfdir); $(RMF) bacula-dir.conf bacula-dir.conf.new) (cd $(DESTDIR)$(scriptdir); $(RMF) query.sql) diff --git a/bacula/src/dird/admin.c b/bacula/src/dird/admin.c index 0f3505ec1f..9c597d7054 100644 --- a/bacula/src/dird/admin.c +++ b/bacula/src/dird/admin.c @@ -73,7 +73,7 @@ void admin_cleanup(JCR *jcr, int TermCode) int msg_type; MEDIA_DBR mr; - Dmsg0(100, "Enter backup_cleanup()\n"); + Dmsg0(100, "Enter admin_cleanup()\n"); update_job_end(jcr, TermCode); diff --git a/bacula/src/dird/authenticate.c b/bacula/src/dird/authenticate.c index 0dce7ef5db..023887cd5d 100644 --- a/bacula/src/dird/authenticate.c +++ b/bacula/src/dird/authenticate.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -37,8 +37,9 @@ extern DIRRES *director; /* Version at end of Hello * prior to 06Aug13 no version * 102 04Jun15 - added jobmedia change + * 103 14Feb17 - added comm line compression */ -#define DIR_VERSION 102 +#define DIR_VERSION 103 /* Command sent to SD */ @@ -168,7 +169,14 @@ bool authenticate_storage_daemon(JCR *jcr, STORE *store) sd->host(), sd->port()); return 0; } - if (jcr->SDVersion < 305) { + /* For newer SD turn on comm line compression */ + if (jcr->SDVersion >= 1 && director->comm_compression) { + sd->set_compress(); + } else { + sd->clear_compress(); + Dmsg0(050, "*** No Dir compression to SD\n"); + } + if (jcr->SDVersion < 2) { Jmsg2(jcr, M_FATAL, 0, _("Older Storage daemon at \"%s:%d\" incompatible with this Director.\n"), sd->host(), sd->port()); return 0; @@ -290,6 +298,13 @@ int authenticate_file_daemon(JCR *jcr) fd->host(), fd->port()); return 0; } + /* For newer FD turn on comm line compression */ + if (jcr->FDVersion >= 9 && director->comm_compression) { + fd->set_compress(); + } else { + fd->clear_compress(); + Dmsg0(050, "*** No Dir compression to FD\n"); + } return 1; } @@ -324,6 +339,13 @@ int authenticate_user_agent(UAContext *uac) return 0; } + /* Turn on compression for newer consoles */ + if (ua_version >= 1 && director->comm_compression) { + ua->set_compress(); + } else { + Dmsg0(050, "*** No Dir compression to UA\n"); + } + name[sizeof(name)-1] = 0; /* terminate name */ if (strcmp(name, "*UserAgent*") == 0) { /* default console */ /* TLS Requirement */ diff --git a/bacula/src/dird/autoprune.c b/bacula/src/dird/autoprune.c index 46416f5229..621cca8201 100644 --- a/bacula/src/dird/autoprune.c +++ b/bacula/src/dird/autoprune.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -17,13 +17,10 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- Automatic Pruning * Applies retention periods * * Kern Sibbald, May MMII - * - * Version $Id$ */ #include "bacula.h" @@ -139,7 +136,7 @@ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr, set_storageid_in_mr(store, mr); if (InChanger) { - Mmsg(changer, "AND InChanger=1 AND StorageId=%s ", edit_int64(mr->StorageId, ed3)); + Mmsg(changer, "AND InChanger=1 AND StorageId IN (%s) ", mr->sid_group); } Mmsg(query, select, ed1, ed2, mr->MediaType, changer.c_str()); @@ -156,7 +153,7 @@ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr, for (i=0; idb, &lmr)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db)); continue; @@ -170,14 +167,14 @@ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr, /* Prune only Volumes with status "Full", or "Used" */ if (strcmp(lmr.VolStatus, "Full") == 0 || strcmp(lmr.VolStatus, "Used") == 0) { - Dmsg2(100, "Add prune list MediaId=%d Volume %s\n", (int)lmr.MediaId, lmr.VolumeName); + Dmsg2(100, "Add prune list MediaId=%lu Volume %s\n", lmr.MediaId, lmr.VolumeName); count = get_prune_list_for_volume(ua, &lmr, &prune_list); Dmsg1(100, "Num pruned = %d\n", count); if (count != 0) { purge_job_list_from_catalog(ua, prune_list); prune_list.num_ids = 0; /* reset count */ } - if (!is_volume_purged(ua, &lmr, false)) { + if (!is_volume_purged(ua, &lmr)) { Dmsg1(100, "Vol=%s not pruned\n", lmr.VolumeName); continue; } @@ -189,6 +186,7 @@ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr, * If not, just skip this volume and try the next one */ if (InChanger) { + /* ***FIXME*** should be any StorageId in sid_group */ if (!lmr.InChanger || (lmr.StorageId != mr->StorageId)) { Dmsg1(100, "Vol=%s not inchanger\n", lmr.VolumeName); continue; /* skip this volume, ie not loadable */ @@ -200,12 +198,17 @@ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr, continue; } + if (has_volume_expired(jcr, &lmr)) { + Dmsg1(100, "Vol=%s has expired\n", lmr.VolumeName); + continue; /* Volume not usable */ + } + /* * If purged and not moved to another Pool, * then we stop pruning and take this volume. */ if (lmr.PoolId == mr->PoolId) { - Dmsg2(100, "Got Vol=%s MediaId=%d purged.\n", lmr.VolumeName, (int)lmr.MediaId); + Dmsg2(100, "Got Vol=%s MediaId=%lu purged.\n", lmr.VolumeName, lmr.MediaId); mr->copy(&lmr); set_storageid_in_mr(store, mr); break; /* got a volume */ diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 6e4c7c88cd..055b60f833 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- backup.c -- responsible for doing backup jobs * * Kern Sibbald, March MM @@ -28,7 +27,6 @@ * Open connection with File daemon and pass him commands * to do the backup. * When the File daemon finishes the job, update the DB. - * */ #include "bacula.h" @@ -42,6 +40,12 @@ static char storaddr[] = "storage address=%s port=%d ssl=%d\n"; /* Responses received from File daemon */ static char OKbackup[] = "2000 OK backup\n"; static char OKstore[] = "2000 OK storage\n"; +/* After 17 Aug 2013 */ +static char newEndJob[] = "2800 End Job TermCode=%d JobFiles=%u " + "ReadBytes=%llu JobBytes=%llu Errors=%u " + "VSS=%d Encrypt=%d " + "CommBytes=%lld CompressCommBytes=%lld\n"; +/* Pre 17 Aug 2013 */ static char EndJob[] = "2800 End Job TermCode=%d JobFiles=%u " "ReadBytes=%llu JobBytes=%llu Errors=%u " "VSS=%d Encrypt=%d\n"; @@ -64,19 +68,19 @@ bool do_backup_init(JCR *jcr) /* Make local copy */ jcr->RescheduleIncompleteJobs = jcr->job->RescheduleIncompleteJobs; + if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { + return do_vbackup_init(jcr); + } + free_rstorage(jcr); /* we don't read so release */ + if (!get_or_create_fileset_record(jcr)) { - Dmsg1(100, "JobId=%d no FileSet\n", (int)jcr->JobId); return false; } /* * Get definitive Job level and since time - * unless it's a virtual full. In that case - * it is not needed. */ - if (!jcr->is_JobLevel(L_VIRTUAL_FULL)) { - get_level_since_time(jcr, jcr->since, sizeof(jcr->since)); - } + get_level_since_time(jcr, jcr->since, sizeof(jcr->since)); apply_pool_overrides(jcr); @@ -86,22 +90,9 @@ bool do_backup_init(JCR *jcr) jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name()); if (jcr->jr.PoolId == 0) { - Dmsg1(100, "JobId=%d no PoolId\n", (int)jcr->JobId); - Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n")); return false; } - /* - * If we are a virtual full job or got upgraded to one - * then we divert at this point and call the virtual full - * backup init method - */ - if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { - return do_vbackup_init(jcr); - } - - free_rstorage(jcr); /* we don't read so release */ - /* If pool storage specified, use it instead of job storage */ copy_wstorage(jcr, jcr->pool->storage, _("Pool resource")); @@ -267,10 +258,15 @@ bool send_accurate_current_files(JCR *jcr) /* On Full mode, if no previous base job, no accurate things */ if (get_base_jobids(jcr, &jobids)) { jcr->HasBase = true; - Jmsg(jcr, M_INFO, 0, _("Using Base JobId(s): %s\n"), jobids.list); + Jmsg(jcr, M_INFO, 0, _("Using BaseJobId(s): %s\n"), jobids.list); } else if (!jcr->rerunning) { return true; } + + } else if (jcr->is_JobLevel(L_VERIFY_DATA)) { + char ed1[50]; + jobids.add(edit_uint64(jcr->previous_jr.JobId, ed1)); + } else { /* For Incr/Diff level, we search for older jobs */ db_get_accurate_jobids(jcr, jcr->db, &jcr->jr, &jobids); @@ -372,7 +368,7 @@ bool send_client_addr_to_sd(JCR *jcr) /* * Send Client address to the SD */ - sd->fsend(clientaddr, jcr->client->address, jcr->client->FDport, tls_need); + sd->fsend(clientaddr, jcr->client->address(), jcr->client->FDport, tls_need); if (!response(jcr, sd, OKclient, "Client", DISPLAY_ERROR)) { return false; } @@ -672,6 +668,8 @@ int wait_for_job_termination(JCR *jcr, int timeout) uint32_t JobWarnings = 0; uint64_t ReadBytes = 0; uint64_t JobBytes = 0; + uint64_t CommBytes = 0; + uint64_t CommCompressedBytes = 0; int VSS = 0; /* or Snapshot on Unix */ int Encrypt = 0; btimer_t *tid=NULL; @@ -682,10 +680,13 @@ int wait_for_job_termination(JCR *jcr, int timeout) } /* Wait for Client to terminate */ while ((n = bget_dirmsg(fd)) >= 0) { - if (!fd_ok && - (sscanf(fd->msg, EndJob, &jcr->FDJobStatus, &JobFiles, + if (!fd_ok && + (sscanf(fd->msg, newEndJob, &jcr->FDJobStatus, &JobFiles, + &ReadBytes, &JobBytes, &JobErrors, &VSS, &Encrypt, + &CommBytes, &CommCompressedBytes) == 9 || + sscanf(fd->msg, EndJob, &jcr->FDJobStatus, &JobFiles, &ReadBytes, &JobBytes, &JobErrors, &VSS, &Encrypt) == 7 || - sscanf(fd->msg, OldEndJob, &jcr->FDJobStatus, &JobFiles, + sscanf(fd->msg, OldEndJob, &jcr->FDJobStatus, &JobFiles, &ReadBytes, &JobBytes, &JobErrors) == 5)) { fd_ok = true; jcr->setJobStatus(jcr->FDJobStatus); @@ -735,6 +736,8 @@ int wait_for_job_termination(JCR *jcr, int timeout) jcr->ReadBytes = ReadBytes; jcr->JobBytes = JobBytes; jcr->JobWarnings = JobWarnings; + jcr->CommBytes = CommBytes; + jcr->CommCompressedBytes = CommCompressedBytes; jcr->Snapshot = VSS; jcr->Encrypt = Encrypt; } else if (jcr->getJobStatus() != JS_Canceled) { @@ -766,9 +769,9 @@ void backup_cleanup(JCR *jcr, int TermCode) char sdt[50], edt[50], schedt[50]; char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30]; char ec6[30], ec7[30], ec8[30], ec9[30], ec10[30], elapsed[50]; - char data_compress[200]; - char term_code[100], fd_term_msg[100], sd_term_msg[100]; - const char *term_msg; + char data_compress[200], comm_compress[200]; + char fd_term_msg[100], sd_term_msg[100]; + POOL_MEM term_msg; int msg_type = M_INFO; MEDIA_DBR mr; CLIENT_DBR cr; @@ -777,6 +780,8 @@ void backup_cleanup(JCR *jcr, int TermCode) POOL_MEM base_info; POOL_MEM vol_info; + remove_dummy_jobmedia_records(jcr); + if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { vbackup_cleanup(jcr, TermCode); return; @@ -785,6 +790,18 @@ void backup_cleanup(JCR *jcr, int TermCode) Dmsg2(100, "Enter backup_cleanup %d %c\n", TermCode, TermCode); memset(&cr, 0, sizeof(cr)); +#ifdef xxxx + /* The current implementation of the JS_Warning status is not + * completed. SQL part looks to be ok, but the code is using + * JS_Terminated almost everywhere instead of (JS_Terminated || JS_Warning) + * as we do with is_canceled() + */ + if (jcr->getJobStatus() == JS_Terminated && + (jcr->JobErrors || jcr->SDErrors || jcr->JobWarnings)) { + TermCode = JS_Warnings; + } +#endif + update_job_end(jcr, TermCode); if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) { @@ -811,20 +828,21 @@ void backup_cleanup(JCR *jcr, int TermCode) switch (jcr->JobStatus) { case JS_Terminated: if (jcr->JobErrors || jcr->SDErrors) { - term_msg = _("Backup OK -- with warnings"); + Mmsg(term_msg, _("Backup OK -- %s"), jcr->StatusErrMsg[0] ? jcr->StatusErrMsg : _("with warnings")); + } else { - term_msg = _("Backup OK"); + Mmsg(term_msg, _("Backup OK")); } break; case JS_Incomplete: - term_msg = _("Backup failed -- incomplete"); + Mmsg(term_msg, _("Backup failed -- incomplete")); break; case JS_Warnings: - term_msg = _("Backup OK -- with warnings"); + Mmsg(term_msg, _("Backup OK -- %s"), jcr->StatusErrMsg[0] ? jcr->StatusErrMsg : _("with warnings")); break; case JS_FatalError: case JS_ErrorTerminated: - term_msg = _("*** Backup Error ***"); + Mmsg(term_msg, _("*** Backup Error ***")); msg_type = M_ERROR; /* Generate error message */ if (jcr->store_bsock) { jcr->store_bsock->signal(BNET_TERMINATE); @@ -834,7 +852,7 @@ void backup_cleanup(JCR *jcr, int TermCode) } break; case JS_Canceled: - term_msg = _("Backup Canceled"); + Mmsg(term_msg, _("Backup Canceled")); if (jcr->store_bsock) { jcr->store_bsock->signal(BNET_TERMINATE); if (jcr->SD_msg_chan_started) { @@ -843,8 +861,7 @@ void backup_cleanup(JCR *jcr, int TermCode) } break; default: - term_msg = term_code; - sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus); + Mmsg(term_msg, _("Inappropriate term code: %c\n"), jcr->JobStatus); break; } bstrftimes(schedt, sizeof(schedt), jcr->jr.SchedTime); @@ -852,10 +869,9 @@ void backup_cleanup(JCR *jcr, int TermCode) bstrftimes(edt, sizeof(edt), jcr->jr.EndTime); RunTime = jcr->jr.EndTime - jcr->jr.StartTime; if (RunTime <= 0) { - kbps = 0; - } else { - kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime); + RunTime = 1; } + kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime); if (!db_get_job_volume_names(jcr, jcr->db, jcr->jr.JobId, &jcr->VolumeName)) { /* * Note, if the job has erred, most likely it did not write any @@ -885,6 +901,20 @@ void backup_cleanup(JCR *jcr, int TermCode) compression, ratio); } } + if (jcr->CommBytes == 0 || jcr->CommCompressedBytes == 0) { + bstrncpy(comm_compress, "None", sizeof(comm_compress)); + } else { + compression = (double)100 - 100.0 * ((double)jcr->CommCompressedBytes / (double)jcr->CommBytes); + if (compression < 0.5) { + bstrncpy(comm_compress, "None", sizeof(comm_compress)); + } else { + ratio = (double)jcr->CommBytes / (double)jcr->CommCompressedBytes; + bsnprintf(comm_compress, sizeof(comm_compress), "%.1f%% %.1f:1", + compression, ratio); + } + Dmsg2(200, "=== CommCompressed=%lld CommBytes=%lld\n", + jcr->CommCompressedBytes, jcr->CommBytes); + } jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg)); jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); @@ -930,6 +960,7 @@ void backup_cleanup(JCR *jcr, int TermCode) " SD Bytes Written: %s (%sB)\n" " Rate: %.1f KB/s\n" " Software Compression: %s\n" +" Comm Line Compression: %s\n" "%s" /* Basefile info */ " Snapshot/VSS: %s\n" " Encryption: %s\n" @@ -966,6 +997,7 @@ void backup_cleanup(JCR *jcr, int TermCode) edit_uint64_with_suffix(jcr->SDJobBytes, ec6), kbps, data_compress, + comm_compress, base_info.c_str(), jcr->Snapshot?_("yes"):_("no"), jcr->Encrypt?_("yes"):_("no"), @@ -978,7 +1010,7 @@ void backup_cleanup(JCR *jcr, int TermCode) jcr->SDErrors, fd_term_msg, sd_term_msg, - term_msg); + term_msg.c_str()); Dmsg0(100, "Leave backup_cleanup()\n"); } @@ -1004,7 +1036,7 @@ void update_bootstrap_file(JCR *jcr) fd = bpipe ? bpipe->wfd : NULL; } else { /* ***FIXME*** handle BASE */ - fd = fopen(fname, jcr->is_JobLevel(L_FULL)?"w+b":"a+b"); + fd = bfopen(fname, jcr->is_JobLevel(L_FULL)?"w+b":"a+b"); } if (fd) { VolCount = db_get_job_volume_parameters(jcr, jcr->db, jcr->JobId, diff --git a/bacula/src/dird/bacula-dir.conf.in b/bacula/src/dird/bacula-dir.conf.in index 85becdf702..a9b7f04a5b 100644 --- a/bacula/src/dird/bacula-dir.conf.in +++ b/bacula/src/dird/bacula-dir.conf.in @@ -11,10 +11,10 @@ # from root to your address. See the "mail" and "operator" # directives in the Messages resource. # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - + Director { # define myself Name = @basename@-dir DIRport = @dir_port@ # where we listen for UA connections @@ -197,7 +197,7 @@ Client { # Definition of file Virtual Autochanger device -Storage { +Autochanger { Name = File1 # Do not use "localhost" here Address = @hostname@ # N.B. Use a fully qualified name here @@ -206,11 +206,12 @@ Storage { Device = FileChgr1 Media Type = File1 Maximum Concurrent Jobs = 10 # run up to 10 jobs a the same time + Autochanger = File1 # point to ourself } # Definition of a second file Virtual Autochanger device # Possibly pointing to a different disk drive -Storage { +Autochanger { Name = File2 # Do not use "localhost" here Address = @hostname@ # N.B. Use a fully qualified name here @@ -218,11 +219,12 @@ Storage { Password = "@sd_password@" Device = FileChgr2 Media Type = File2 + Autochanger = File2 # point to ourself Maximum Concurrent Jobs = 10 # run up to 10 jobs a the same time } # Definition of LTO-4 tape Autochanger device -#Storage { +#Autochanger { # Name = LTO-4 # Do not use "localhost" here # Address = @hostname@ # N.B. Use a fully qualified name here @@ -230,6 +232,7 @@ Storage { # Password = "@sd_password@" # password for Storage daemon # Device = LTO-4 # must be same as Device in Storage daemon # Media Type = LTO-4 # must be same as MediaType in Storage daemon +# Autochanger = LTO-4 # enable for autochanger device # Maximum Concurrent Jobs = 10 #} diff --git a/bacula/src/dird/bdirjson.c b/bacula/src/dird/bdirjson.c new file mode 100644 index 0000000000..ef5bf10c20 --- /dev/null +++ b/bacula/src/dird/bdirjson.c @@ -0,0 +1,1463 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * Bacula Director conf to Json + * + * Kern Sibbald, September MMXII + * + */ + +#include "bacula.h" +#include "dird.h" + +/* Exported subroutines */ +extern bool parse_dir_config(CONFIG *config, const char *configfile, int exit_code); + +static CONFIG *config; + +/* Globals Exported */ +DIRRES *director; /* Director resource */ +int FDConnectTimeout; +int SDConnectTimeout; +char *configfile = NULL; +void *start_heap; + +/* Globals Imported */ +extern RES_ITEM job_items[]; +extern s_jt jobtypes[]; +extern s_jl joblevels[]; +extern s_jt migtypes[]; +extern s_kw ReplaceOptions[]; +extern RES_ITEM2 newinc_items[]; +extern RES_ITEM options_items[]; +extern s_fs_opt FS_options[]; +extern s_kw RunFields[]; +extern s_kw tapelabels[]; +extern s_kw msg_types[]; +extern RES_TABLE resources[]; + +#if defined(_MSC_VER) +extern "C" { // work around visual compiler mangling variables + extern URES res_all; +} +#else +extern URES res_all; +#endif + + +#define CONFIG_FILE "bacula-dir.conf" /* default configuration file */ + +static void usage() +{ + fprintf(stderr, _( +PROG_COPYRIGHT +"\n%sVersion: %s (%s)\n\n" +"Usage: bdirjson [] [config_file]\n" +" -r get resource type \n" +" -n get resource \n" +" -l get only directives matching dirs (use with -r)\n" +" -D get only data\n" +" -R do not apply JobDefs to Job\n" +" -c set configuration file to file\n" +" -d set debug level to \n" +" -dt print timestamp in debug output\n" +" -t test - read configuration and exit\n" +" -s output in show text format\n" +" -v verbose user messages\n" +" -? print this message.\n" +"\n"), 2012, "", VERSION, BDATE); + + exit(1); +} + +typedef struct +{ + /* default { { "Director": { "Name": aa, ...} }, { "Job": {..} */ + bool do_list; /* [ {}, {}, ..] or { "aa": {}, "bb": {}, ...} */ + bool do_one; /* { "Name": "aa", "Description": "test, ... } */ + bool do_only_data; /* [ {}, {}, {}, ] */ + char *resource_type; + char *resource_name; + regex_t directive_reg; +} display_filter; + +/* Forward referenced subroutines */ +void terminate_dird(int sig); +static bool check_resources(bool config_test); +static void sendit(void *ua, const char *fmt, ...); +static void dump_json(display_filter *filter); + +/********************************************************************* + * + * Bacula Director conf to Json + * + */ +int main (int argc, char *argv[]) +{ + int ch; + bool test_config = false; + bool apply_jobdefs = true; + bool do_show_format = false; + display_filter filter; + memset(&filter, 0, sizeof(filter)); + + setlocale(LC_ALL, ""); + bindtextdomain("bacula", LOCALEDIR); + textdomain("bacula"); + + my_name_is(argc, argv, "bacula-dir"); + init_msg(NULL, NULL); + + while ((ch = getopt(argc, argv, "RCDc:d:stv?l:r:n:")) != -1) { + switch (ch) { + case 'R': + apply_jobdefs = false; + break; + + case 'D': + filter.do_only_data = true; + break; + + case 'l': + /* Might use something like -l '^(Name|Description)$' */ + filter.do_list = true; + if (regcomp(&filter.directive_reg, optarg, REG_EXTENDED) != 0) { + Jmsg((JCR *)NULL, M_ERROR_TERM, 0, + _("Please use valid -l argument: %s\n"), optarg); + } + break; + + case 'r': + filter.resource_type = optarg; + break; + + case 'n': + filter.resource_name = optarg; + break; + + case 'c': /* specify config file */ + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(optarg); + break; + + case 'd': /* set debug level */ + if (*optarg == 't') { + dbg_timestamp = true; + } else { + debug_level = atoi(optarg); + if (debug_level <= 0) { + debug_level = 1; + } + } + Dmsg1(10, "Debug level = %d\n", debug_level); + break; + + case 's': /* Show text format */ + do_show_format = true; + break; + + case 't': /* test config */ + test_config = true; + break; + + case 'v': /* verbose */ + verbose++; + break; + + case '?': + default: + usage(); + + } + } + argc -= optind; + argv += optind; + + + if (argc) { + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(*argv); + argc--; + argv++; + } + if (argc) { + usage(); + } + + if (filter.do_list && !filter.resource_type) { + usage(); + } + + if (filter.resource_type && filter.resource_name) { + filter.do_one = true; + } + + if (configfile == NULL || configfile[0] == 0) { + configfile = bstrdup(CONFIG_FILE); + } + + if (test_config && verbose > 0) { + char buf[1024]; + find_config_file(configfile, buf, sizeof(buf)); + sendit(NULL, "config_file=%s\n", buf); + } + + config = New(CONFIG()); + config->encode_password(false); + parse_dir_config(config, configfile, M_ERROR_TERM); + + /* TODO: If we run check_resources, jobdefs will be copied to Job, and the job resource + * will no longer be the real job... + */ + if (!check_resources(apply_jobdefs)) { + Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); + } + + if (test_config) { + terminate_dird(0); + } + + my_name_is(0, NULL, director->name()); /* set user defined name */ + + if (do_show_format) { + /* Do show output in text */ + for (int i=r_first; i<=r_last; i++) { + dump_each_resource(i, sendit, NULL); + } + } else { + dump_json(&filter); + } + + if (filter.do_list) { + regfree(&filter.directive_reg); + } + + terminate_dird(0); + + return 0; +} + +/* Cleanup and then exit */ +void terminate_dird(int sig) +{ + static bool already_here = false; + + if (already_here) { /* avoid recursive temination problems */ + bmicrosleep(2, 0); /* yield */ + exit(1); + } + already_here = true; + debug_level = 0; /* turn off debug */ + if (configfile != NULL) { + free(configfile); + } + if (debug_level > 5) { + print_memory_pool_stats(); + } + if (config) { + delete config; + config = NULL; + } + term_msg(); + free(res_head); + res_head = NULL; + close_memory_pool(); /* release free memory in pool */ + //sm_dump(false); + exit(sig); +} + + +static void display_jobtype(HPKT &hpkt) +{ + int i; + for (i=0; jobtypes[i].type_name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == jobtypes[i].job_type) { + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, jobtypes[i].type_name)); + return; + } + } +} + +static void display_label(HPKT &hpkt) +{ + int i; + for (i=0; tapelabels[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == tapelabels[i].token) { + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, tapelabels[i].name)); + return; + } + } +} + +static void display_joblevel(HPKT &hpkt) +{ + int i; + for (i=0; joblevels[i].level_name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == joblevels[i].level) { + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, joblevels[i].level_name)); + return; + } + } +} + +static void display_replace(HPKT &hpkt) +{ + int i; + for (i=0; ReplaceOptions[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == ReplaceOptions[i].token) { + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, ReplaceOptions[i].name)); + return; + } + } +} + +static void display_migtype(HPKT &hpkt) +{ + int i; + for (i=0; migtypes[i].type_name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == migtypes[i].job_type) { + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, migtypes[i].type_name)); + return; + } + } +} + +static void display_actiononpurge(HPKT &hpkt) +{ + sendit(NULL, "\n \"%s\":", hpkt.ritem->name); + if (*(uint32_t *)(hpkt.ritem->value) | ON_PURGE_TRUNCATE) { + sendit(NULL, "\"Truncate\""); + } else { + sendit(NULL, "null"); + } +} + +static void display_acl(HPKT &hpkt) +{ + sendit(NULL, "\n \"%s\":", hpkt.ritem->name); + hpkt.list = ((alist **)hpkt.ritem->value)[hpkt.ritem->code]; + display_alist(hpkt); +} + + +static void display_options(HPKT &hpkt, INCEXE *ie) +{ + char *elt; + bool first_opt = true; + bool first_dir; + int i, j, k; + alist *list; + + sendit(NULL, " \"Options\": [ \n {\n"); + for (i=0; inum_opts; i++) { + FOPTS *fo = ie->opts_list[i]; + if (!first_opt) { + sendit(NULL, ",\n {\n"); + } + first_dir = true; + for (j=0; options_items[j].name; j++) { + if (options_items[j].handler == store_regex) { + switch (options_items[j].code) { + case 1: /* RegexDir */ + list = &fo->regexdir; + break; + case 2: /* RegexFile */ + list = &fo->regexfile; + break; + default: + list = &fo->regex; + break; + } + if (list->size() > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", options_items[j].name); + hpkt.list = list; + display_alist(hpkt); + first_dir = false; + first_opt = false; + } + } else if (options_items[j].handler == store_wild) { + switch (options_items[j].code) { + case 1: /* WildDir */ + list = &fo->wilddir; + break; + case 2: /* WildFile */ + /* + * Note: There used to be an enhanced wild card feature, + * which was not documented so it is removed, and + * apparently the wildfile patterns are stored in the + * wildbase list, so we dump it here. + * The old enhanced wild implementation appears to be poorly + * done, because either there should be two clearly named + * lists, or one with everything. + */ + /* We copy one list to the other, else we may print two + * times the WildFile list. I don't know how, but sometime + * the two lists contain elements. + */ + list = &fo->wildfile; + foreach_alist(elt, list) { + fo->wildbase.append(bstrdup(elt)); + } + list = &fo->wildbase; + break; + default: + list = &fo->wild; + break; + } + if (list->size() > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", options_items[j].name); + hpkt.list = list; + display_alist(hpkt); + first_dir = false; + first_opt = false; + } + } else if (options_items[j].handler == store_base) { + list = &fo->base; + if (list->size() > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", options_items[j].name); + hpkt.list = list; + display_alist(hpkt); + first_dir = false; + first_opt = false; + } + } else if (options_items[j].handler == store_opts) { + bool found = false; + if (bit_is_set(options_items[j].flags, ie->opt_present)) { + for (k=0; FS_options[k].name; k++) { + if (FS_options[k].keyword == (int)options_items[j].flags) { + char lopts[100]; + strip_long_opts(lopts, fo->opts); + if (strstr(lopts, FS_options[k].option)) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\": %s", options_items[j].name, + quote_string(hpkt.edbuf, FS_options[k].name)); + found = true; + break; + } + } + } + if (found) { + first_dir = false; + first_opt = false; + } + } + } else if (options_items[j].handler == store_lopts) { + bool found = false; + if (bit_is_set(options_items[j].flags, ie->opt_present)) { + char *pos; + /* Search long_options for code (V, J, C, P) */ + if ((pos=strchr(fo->opts, options_items[j].code))) { + char lopts[100]; + char *end, bkp; + pos++; /* point to beginning of options */ + bstrncpy(lopts, pos, sizeof(lopts)); + /* Now terminate at first : */ + end = strchr(pos, ':'); + if (end) { + bkp = *end; /* save the original char */ + *end = 0; /* terminate this string */ + } + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\": %s", options_items[j].name, + quote_string(hpkt.edbuf, pos)); + found = true; + if (end) { /* Still have other options to parse */ + *end = bkp; + } + } + if (found) { + first_dir = false; + first_opt = false; + } + } + } else if (options_items[j].handler == store_plugin) { + if (fo->plugin) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\": %s", options_items[j].name, + quote_string(hpkt.edbuf, fo->plugin)); + first_dir = false; + first_opt = false; + } + } else if (options_items[j].handler == store_fstype) { + list = &fo->fstype; + if (list->size() > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", options_items[j].name); + hpkt.list = list; + display_alist(hpkt); + first_dir = false; + first_opt = false; + } + } else if (options_items[j].handler == store_drivetype) { + list = &fo->drivetype; + if (list->size() > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", options_items[j].name); + hpkt.list = list; + display_alist(hpkt); + first_dir = false; + first_opt = false; + } + } + } + sendit(NULL, "\n }"); + } + sendit(NULL, "\n ]"); +} + +/* + * Include or Exclude in a FileSet + * TODO: Not working with multiple Include{} + * O M + * N + * I /tmp/regress/build + * N + * O Z1 + * N + * I /tmp + * N + */ +static void display_include_exclude(HPKT &hpkt) +{ + bool first_dir; + int i, j; + FILESET *fs = (FILESET *)hpkt.res; + + if (hpkt.ritem->code == 0) { /* Include */ + INCEXE *ie; + sendit(NULL, "\n \"%s\": [{\n", hpkt.ritem->name); + for (j=0; jnum_includes; j++) { + if (j > 0) { + sendit(NULL, ",\n {\n"); + } + first_dir = true; + ie = fs->include_items[j]; + for (i=0; newinc_items[i].name; i++) { + if (strcasecmp(newinc_items[i].name, "File") == 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", newinc_items[i].name); + first_dir = false; + hpkt.list = &ie->name_list; + display_alist(hpkt); + } if (strcasecmp(newinc_items[i].name, "Plugin") == 0 && + ie->plugin_list.size() > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\":", newinc_items[i].name); + first_dir = false; + hpkt.list = &ie->plugin_list; + display_alist(hpkt); + } if (strcasecmp(newinc_items[i].name, "Options") == 0 && + ie->num_opts > 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + display_options(hpkt, ie); + } if (strcasecmp(newinc_items[i].name, "ExcludeDirContaining") == 0 && + ie->ignoredir) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\": %s ", newinc_items[i].name, + quote_string(hpkt.edbuf, ie->ignoredir)); + first_dir = false; + } + } + sendit(NULL, "\n }"); + } + sendit(NULL, "]"); + } else { + /* Exclude */ + sendit(NULL, "\n \"%s\": {\n", hpkt.ritem->name); + first_dir = true; + for (int i=0; newinc_items[i].name; i++) { + INCEXE *ie; + if (strcasecmp(newinc_items[i].name, "File") == 0) { + if (!first_dir) { + sendit(NULL, ",\n"); + } + sendit(NULL, " \"%s\": ", newinc_items[i].name); + first_dir = false; + ie = fs->exclude_items[0]; + hpkt.list = &ie->name_list; + display_alist(hpkt); + } + } + sendit(NULL, "\n }"); + } +} + +static bool display_runscript(HPKT &hpkt) +{ + RUNSCRIPT *script; + RUNSCRIPT *def = new_runscript(); + alist **runscripts = (alist **)(hpkt.ritem->value) ; + bool first=true; + + if (!*runscripts || (*runscripts)->size() == 0) { + return false; + } + + sendit(NULL, "\n \"Runscript\": [\n"); + + foreach_alist(script, *runscripts) { + if (first) { + sendit(NULL, " {\n"); + } else { + sendit(NULL, ",\n {\n"); + } + if (script->when == SCRIPT_Any) { + sendit(NULL, " \"RunsWhen\": \"Any\",\n"); + + } else if (script->when == SCRIPT_After) { + sendit(NULL, " \"RunsWhen\": \"After\",\n"); + + } else if (script->when == SCRIPT_Before) { + sendit(NULL, " \"RunsWhen\": \"Before\",\n"); + + } else if (script->when == SCRIPT_AfterVSS) { + sendit(NULL, " \"RunsWhen\": \"AfterVSS\",\n"); + } + + if (script->fail_on_error != def->fail_on_error) { + sendit(NULL, " \"FailJobOnError\": %s,\n", script->fail_on_error?"true":"false"); + } + + if (script->on_success != def->on_success) { + sendit(NULL, " \"RunsOnSuccess\": %s,\n", script->on_success?"true":"false"); + } + + if (script->on_failure != def->on_failure) { + sendit(NULL, " \"RunsOnFailure\": %s,\n", script->on_failure?"true":"false"); + } + + if (script->is_local()) { + sendit(NULL, " \"RunsOnClient\": false,\n"); + } + + if (script->command) { + sendit(NULL, " \"%s\": %s\n", + (script->cmd_type == SHELL_CMD)?"Command":"Console", + quote_string(hpkt.edbuf, script->command)); + } + sendit(NULL, " }"); + first = false; + } + + sendit(NULL, "\n ]\n"); + free_runscript(def); + return true; +} + +static void display_run(HPKT &hpkt) +{ + int i, j; + RUN **prun = (RUN **)hpkt.ritem->value; + RUN *run = *prun; + bool first = true; + bool first_run = true; + RES *res; + + sendit(NULL, "\n \"%s\": [\n", hpkt.ritem->name); + for ( ; run; run=run->next) { + if (!first_run) sendit(NULL, ",\n"); + first_run = false; + first = true; + sendit(NULL, " {\n"); + /* First do override fields */ + for (i=0; RunFields[i].name; i++) { + switch (RunFields[i].token) { + case 'f': /* FullPool */ + if (run->full_pool) { + res = (RES *)run->full_pool; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'i': /* IncrementalPool */ + if (run->inc_pool) { + res = (RES *)run->inc_pool; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'd': /* Differential Pool */ + if (run->diff_pool) { + res = (RES *)run->diff_pool; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'N': /* Next Pool */ + if (run->next_pool) { + res = (RES *)run->next_pool; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'L': /* Level */ + /* TODO: It's not always set, only when having Level= in the line */ + //if (run->level_set) { + for (j=0; joblevels[j].level_name; j++) { + if ((int)run->level == joblevels[j].level) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": \"%s\"", RunFields[i].name, + joblevels[j].level_name); + first = false; + } + } + //} + break; + case 'P': /* Pool */ + if (run->pool) { + res = (RES *)run->pool; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'S': /* Storage */ + if (run->storage) { + res = (RES *)run->storage; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'M': /* Messages */ + if (run->msgs) { + res = (RES *)run->msgs; + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + quote_string(hpkt.edbuf, res->name)); + first = false; + } + break; + case 'p': /* priority */ + if (run->priority_set) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %d", RunFields[i].name, + run->Priority); + first = false; + } + break; + case 's': /* Spool Data */ + if (run->spool_data_set) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + run->spool_data?"true":"false"); + first = false; + } + break; + case 'W': /* Write Part After Job */ + if (run->write_part_after_job_set) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + run->write_part_after_job?"true":"false"); + first = false; + } + break; + case 'm': /* MaxRunScheduledTime */ + if (run->MaxRunSchedTime_set) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %lld", RunFields[i].name, + run->MaxRunSchedTime); + first = false; + } + break; + case 'a': /* Accurate */ + if (run->accurate_set) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"%s\": %s", RunFields[i].name, + run->accurate?"true":"false"); + first = false; + } + break; + default: + break; + } + } /* End all RunFields (overrides) */ + /* Now handle timing */ + if (run->minute) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"Minute\": %d", run->minute); + first = false; + } + if (byte_is_set(run->hour, sizeof(run->hour))) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"Hour\":"); + display_bit_array(run->hour, 24); + first = false; + } + /* bit 32 is used to store the keyword LastDay, so we look up to 0-31 */ + if (byte_is_set(run->mday, sizeof(run->mday) - 1)) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"Day\":"); + display_bit_array(run->mday, 31); + first = false; + } + if (run->last_day_set) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"LastDay\": 1"); + first = false; + } + if (byte_is_set(run->month, sizeof(run->month))) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"Month\":"); + display_bit_array(run->month, 12); + first = false; + } + if (byte_is_set(run->wday, sizeof(run->wday))) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"DayOfWeek\":"); + display_bit_array(run->wday, 7); + first = false; + } + if (byte_is_set(run->wom, sizeof(run->wom))) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"WeekOfMonth\":"); + display_bit_array(run->wom, 5); + first = false; + } + if (byte_is_set(run->woy, sizeof(run->woy))) { + if (!first) sendit(NULL, ",\n"); + sendit(NULL, " \"WeekOfYear\":"); + display_bit_array(run->woy, 54); + first = false; + } + sendit(NULL, "\n }"); + + } /* End this Run directive */ + sendit(NULL, "\n ]"); +} + +/* + * Dump out all resources in json format. + * Note!!!! This routine must be in this file rather + * than in src/lib/parser_conf.c otherwise the pointers + * will be all messed up. + */ +static void dump_json(display_filter *filter) +{ + int resinx, item, first_directive; + bool first_res; + RES_ITEM *items; + RES *res; + HPKT hpkt; + regmatch_t pmatch[32]; + + init_hpkt(hpkt); + + /* List resources and directives */ + if (filter->do_only_data) { + /* Skip the Name */ + sendit(NULL, "["); + + /* + * { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } + * or print a single item + */ + } else if (filter->do_one || filter->do_list) { + sendit(NULL, "{"); + + } else { + /* [ { "Client": { "Name": "aa",.. } }, { "Director": { "Name": "bb", ... } } ]*/ + sendit(NULL, "["); + } + + first_res = true; + /* Main loop over all resources */ + for (resinx=0; resources[resinx].items; resinx++) { + + /* Skip this resource type? */ + if (filter->resource_type && + strcasecmp(filter->resource_type, resources[resinx].name) != 0) { + continue; + } + + /* Loop over each resource of this type */ + foreach_rblist(res, res_head[resinx]->res_list) { + hpkt.res = res; + items = resources[resinx].items; + if (!items) { + continue; + } + + /* Copy the resource into res_all */ + memcpy(&res_all, res, sizeof(res_all)); + + /* If needed, skip this resource type */ + if (filter->resource_name) { + bool skip=true; + /* The Name should be at the first place, so this is not a real loop */ + for (item=0; items[item].name; item++) { + if (strcasecmp(items[item].name, "Name") == 0) { + if (strcmp(*(items[item].value), filter->resource_name) == 0) { + skip = false; + } + break; + } + } + if (skip) { /* The name doesn't match, so skip it */ + continue; + } + } + + if (first_res) { + sendit(NULL, "\n"); + } else { + sendit(NULL, ",\n"); + } + + if (filter->do_only_data) { + sendit(NULL, " {"); + + } else if (filter->do_one) { + /* Nothing to print */ + + /* When sending the list, the form is: + * { aa: { Name: aa, Description: aadesc...}, bb: { Name: bb + */ + } else if (filter->do_list) { + /* Search and display Name, should be the first item */ + for (item=0; items[item].name; item++) { + if (strcmp(items[item].name, "Name") == 0) { + sendit(NULL, "%s: {\n", quote_string(hpkt.edbuf2, *items[item].value)); + break; + } + } + } else { + /* Begin new resource */ + sendit(NULL, "{\n \"%s\": {", resources[resinx].name); + } + + first_res = false; + first_directive = 0; + + /* + * Here we walk through a resource displaying all the + * directives and sub-resources in the resource. + */ + for (item=0; items[item].name; item++) { + /* Check user argument -l */ + if (filter->do_list && + regexec(&filter->directive_reg, + items[item].name, 32, pmatch, 0) != 0) + { + continue; + } + + hpkt.ritem = &items[item]; + + if (bit_is_set(item, res_all.hdr.item_present)) { + + /* Skip Directive in lowercase, but check if the next + * one is pointing to the same location (for example User and dbuser) + */ + if (!B_ISUPPER(*(items[item].name))) { + int i=item+1; + while(!B_ISUPPER(*(items[i].name)) && items[i].value == items[item].value) { + i++; + } + if (items[i].value == items[item].value) { + set_bit(i, res_all.hdr.item_present); + } + continue; + } + + if (first_directive++ > 0) sendit(NULL, ","); + if (display_global_item(hpkt)) { + /* Fall-through wanted */ + } else if (items[item].handler == store_jobtype) { + display_jobtype(hpkt); + } else if (items[item].handler == store_label) { + display_label(hpkt); + } else if (items[item].handler == store_level) { + display_joblevel(hpkt); + } else if (items[item].handler == store_replace) { + display_replace(hpkt); + } else if (items[item].handler == store_migtype) { + display_migtype(hpkt); + } else if (items[item].handler == store_actiononpurge) { + display_actiononpurge(hpkt); + /* FileSet Include/Exclude directive */ + } else if (items[item].handler == store_inc) { + display_include_exclude(hpkt); + } else if (items[item].handler == store_ac_res) { + display_res(hpkt); + /* A different alist for each item.code */ + } else if (items[item].handler == store_acl) { + display_acl(hpkt); + } else if (items[item].handler == store_device) { + display_alist_res(hpkt); + } else if (items[item].handler == store_run) { + display_run(hpkt); + } else if (items[item].handler == store_runscript) { + if (!display_runscript(hpkt)) { + first_directive = 0; /* Do not print a comma after this empty runscript */ + } + } else { + sendit(NULL, "\n \"%s\": null", items[item].name); + } + } else { /* end if is present */ + /* For some directive, the bitmap is not set (like addresses) */ + if (strcmp(resources[resinx].name, "Director") == 0) { + if (strcmp(items[item].name, "DirPort") == 0) { + if (get_first_port_host_order(director->DIRaddrs) != items[item].default_value) { + if (first_directive++ > 0) sendit(NULL, ","); + sendit(NULL, "\n \"DirPort\": %d", + get_first_port_host_order(director->DIRaddrs)); + } + + } else if (strcmp(items[item].name, "DirAddress") == 0) { + char buf[500]; + get_first_address(director->DIRaddrs, buf, sizeof(buf)); + if (strcmp(buf, "0.0.0.0") != 0) { + if (first_directive++ > 0) sendit(NULL, ","); + sendit(NULL, "\n \"DirAddress\": \"%s\"", buf); + } + + } else if (strcmp(items[item].name, "DirSourceAddress") == 0 && director->DIRsrc_addr) { + char buf[500]; + get_first_address(director->DIRsrc_addr, buf, sizeof(buf)); + if (strcmp(buf, "0.0.0.0") != 0) { + if (first_directive++ > 0) sendit(NULL, ","); + sendit(NULL, "\n \"DirSourceAddress\": \"%s\"", buf); + } + } + } + } + if (items[item].flags & ITEM_LAST) { + display_last(hpkt); /* If last bit set always call to cleanup */ + } + } /* loop over directive names */ + + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } */ + if (filter->do_only_data || filter->do_list) { + sendit(NULL, "\n }"); /* Finish the Resource with a single } */ + + } else { + if (filter->do_one) { + /* don't print anything */ + + } else if (first_directive > 0) { + sendit(NULL, "\n }\n}"); /* end of resource */ + + } else { + sendit(NULL, "}\n}"); + } + } + } /* End loop over all resources of this type */ + } /* End loop all resource types */ + + if (filter->do_only_data) { + sendit(NULL, "\n]\n"); + + /* In list context, we are dealing with a hash */ + } else if (filter->do_one || filter->do_list) { + sendit(NULL, "\n}\n"); + + } else { + sendit(NULL, "\n]\n"); + } + term_hpkt(hpkt); +} + +/* + * Make a quick check to see that we have all the + * resources needed. + * + * **** FIXME **** this routine could be a lot more + * intelligent and comprehensive. + */ +static bool check_resources(bool apply_jobdefs) +{ + bool OK = true; + JOB *job; + bool need_tls; + + LockRes(); + + job = (JOB *)GetNextRes(R_JOB, NULL); + director = (DIRRES *)GetNextRes(R_DIRECTOR, NULL); + if (!director) { + Jmsg(NULL, M_FATAL, 0, _("No Director resource defined in %s\n" +"Without that I don't know who I am :-(\n"), configfile); + OK = false; + } else { + set_working_directory(director->working_directory); + if (!director->messages) { /* If message resource not specified */ + director->messages = (MSGS *)GetNextRes(R_MSGS, NULL); + if (!director->messages) { + Jmsg(NULL, M_FATAL, 0, _("No Messages resource defined in %s\n"), configfile); + OK = false; + } + } + if (GetNextRes(R_DIRECTOR, (RES *)director) != NULL) { + Jmsg(NULL, M_FATAL, 0, _("Only one Director resource permitted in %s\n"), + configfile); + OK = false; + } + /* tls_require implies tls_enable */ + if (director->tls_require) { + if (have_tls) { + director->tls_enable = true; + } else { + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + } + } + + need_tls = director->tls_enable || director->tls_authenticate; + + if (!director->tls_certfile && need_tls) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Director \"%s\" in %s.\n"), + director->name(), configfile); + OK = false; + } + + if (!director->tls_keyfile && need_tls) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Key\" file not defined for Director \"%s\" in %s.\n"), + director->name(), configfile); + OK = false; + } + + if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && + need_tls && director->tls_verify_peer) { + Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\" or \"TLS CA" + " Certificate Dir\" are defined for Director \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + director->name(), configfile); + OK = false; + } + } + + /* Loop over Consoles */ + CONRES *cons; + foreach_res(cons, R_CONSOLE) { + /* tls_require implies tls_enable */ + if (cons->tls_require) { + if (have_tls) { + cons->tls_enable = true; + } else { + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; + } + } + + need_tls = cons->tls_enable || cons->tls_authenticate; + + if (!cons->tls_certfile && need_tls) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Console \"%s\" in %s.\n"), + cons->name(), configfile); + OK = false; + } + + if (!cons->tls_keyfile && need_tls) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Key\" file not defined for Console \"%s\" in %s.\n"), + cons->name(), configfile); + OK = false; + } + + if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) + && need_tls && cons->tls_verify_peer) { + Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\" or \"TLS CA" + " Certificate Dir\" are defined for Console \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + cons->name(), configfile); + OK = false; + } + /* If everything is well, attempt to initialize our per-resource TLS context */ + if (OK && (need_tls || cons->tls_require)) { + /* Initialize TLS context: + * Args: CA certfile, CA certdir, Certfile, Keyfile, + * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */ + cons->tls_ctx = new_tls_context(cons->tls_ca_certfile, + cons->tls_ca_certdir, cons->tls_certfile, + cons->tls_keyfile, NULL, NULL, cons->tls_dhfile, cons->tls_verify_peer); + + if (!cons->tls_ctx) { + Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for File daemon \"%s\" in %s.\n"), + cons->name(), configfile); + OK = false; + } + } + + } + + /* Loop over Clients */ + CLIENT *client; + foreach_res(client, R_CLIENT) { + /* tls_require implies tls_enable */ + if (client->tls_require) { + if (have_tls) { + client->tls_enable = true; + } else { + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; + } + } + need_tls = client->tls_enable || client->tls_authenticate; + if ((!client->tls_ca_certfile && !client->tls_ca_certdir) && need_tls) { + Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for File daemon \"%s\" in %s.\n"), + client->name(), configfile); + OK = false; + } + + } + + if (!job) { + Jmsg(NULL, M_FATAL, 0, _("No Job records defined in %s\n"), configfile); + OK = false; + } + + /* TODO: We can't really update all job, we need to show only the real configuration + * and not Job+JobDefs + */ + if (!apply_jobdefs) { + UnlockRes(); + return OK; + } + + foreach_res(job, R_JOB) { + int i; + + if (job->jobdefs) { + JOB *jobdefs = job->jobdefs; + /* Handle RunScripts alists specifically */ + if (jobdefs->RunScripts) { + RUNSCRIPT *rs, *elt; + + if (!job->RunScripts) { + job->RunScripts = New(alist(10, not_owned_by_alist)); + } + + foreach_alist(rs, jobdefs->RunScripts) { + elt = copy_runscript(rs); + job->RunScripts->append(elt); /* we have to free it */ + } + } + + /* Transfer default items from JobDefs Resource */ + for (i=0; job_items[i].name; i++) { + char **def_svalue, **svalue; /* string value */ + uint32_t *def_ivalue, *ivalue; /* integer value */ + bool *def_bvalue, *bvalue; /* bool value */ + int64_t *def_lvalue, *lvalue; /* 64 bit values */ + uint32_t offset; + alist **def_avalue, **avalue; /* alist value */ + + Dmsg4(1400, "Job \"%s\", field \"%s\" bit=%d def=%d\n", + job->name(), job_items[i].name, + bit_is_set(i, job->hdr.item_present), + bit_is_set(i, job->jobdefs->hdr.item_present)); + + if (!bit_is_set(i, job->hdr.item_present) && + bit_is_set(i, job->jobdefs->hdr.item_present)) { + Dmsg2(400, "Job \"%s\", field \"%s\": getting default.\n", + job->name(), job_items[i].name); + offset = (char *)(job_items[i].value) - (char *)&res_all; + /* + * Handle strings and directory strings + */ + if (job_items[i].handler == store_str || + job_items[i].handler == store_dir) { + def_svalue = (char **)((char *)(job->jobdefs) + offset); + Dmsg5(400, "Job \"%s\", field \"%s\" def_svalue=%s item %d offset=%u\n", + job->name(), job_items[i].name, *def_svalue, i, offset); + svalue = (char **)((char *)job + offset); + if (*svalue) { + Pmsg1(000, _("Hey something is wrong. p=0x%lu\n"), *svalue); + } + *svalue = bstrdup(*def_svalue); + set_bit(i, job->hdr.item_present); + /* + * Handle resources + */ + } else if (job_items[i].handler == store_res) { + def_svalue = (char **)((char *)(job->jobdefs) + offset); + Dmsg4(400, "Job \"%s\", field \"%s\" item %d offset=%u\n", + job->name(), job_items[i].name, i, offset); + svalue = (char **)((char *)job + offset); + if (*svalue) { + Pmsg1(000, _("Hey something is wrong. p=0x%lu\n"), *svalue); + } + *svalue = *def_svalue; + set_bit(i, job->hdr.item_present); + /* + * Handle alist resources + */ + } else if (job_items[i].handler == store_alist_str) { + char *elt; + + def_avalue = (alist **)((char *)(job->jobdefs) + offset); + avalue = (alist **)((char *)job + offset); + + *avalue = New(alist(10, owned_by_alist)); + + foreach_alist(elt, (*def_avalue)) { + (*avalue)->append(bstrdup(elt)); + } + set_bit(i, job->hdr.item_present); + + } else if (job_items[i].handler == store_alist_res) { + void *elt; + + def_avalue = (alist **)((char *)(job->jobdefs) + offset); + avalue = (alist **)((char *)job + offset); + + *avalue = New(alist(10, not_owned_by_alist)); + + foreach_alist(elt, (*def_avalue)) { + (*avalue)->append(elt); + } + set_bit(i, job->hdr.item_present); + /* + * Handle integer fields + * Note, our store_bit does not handle bitmaped fields + */ + } else if (job_items[i].handler == store_bit || + job_items[i].handler == store_pint32 || + job_items[i].handler == store_jobtype || + job_items[i].handler == store_level || + job_items[i].handler == store_int32 || + job_items[i].handler == store_size32 || + job_items[i].handler == store_migtype || + job_items[i].handler == store_replace) { + def_ivalue = (uint32_t *)((char *)(job->jobdefs) + offset); + Dmsg5(400, "Job \"%s\", field \"%s\" def_ivalue=%d item %d offset=%u\n", + job->name(), job_items[i].name, *def_ivalue, i, offset); + ivalue = (uint32_t *)((char *)job + offset); + *ivalue = *def_ivalue; + set_bit(i, job->hdr.item_present); + /* + * Handle 64 bit integer fields + */ + } else if (job_items[i].handler == store_time || + job_items[i].handler == store_size64 || + job_items[i].handler == store_int64) { + def_lvalue = (int64_t *)((char *)(job->jobdefs) + offset); + Dmsg5(400, "Job \"%s\", field \"%s\" def_lvalue=%" lld " item %d offset=%u\n", + job->name(), job_items[i].name, *def_lvalue, i, offset); + lvalue = (int64_t *)((char *)job + offset); + *lvalue = *def_lvalue; + set_bit(i, job->hdr.item_present); + /* + * Handle bool fields + */ + } else if (job_items[i].handler == store_bool) { + def_bvalue = (bool *)((char *)(job->jobdefs) + offset); + Dmsg5(400, "Job \"%s\", field \"%s\" def_bvalue=%d item %d offset=%u\n", + job->name(), job_items[i].name, *def_bvalue, i, offset); + bvalue = (bool *)((char *)job + offset); + *bvalue = *def_bvalue; + set_bit(i, job->hdr.item_present); + + } else { + Dmsg1(10, "Handler missing for job_items[%d]\n", i); + ASSERTD(0, "JobDefs -> Job handler missing\n"); + } + } + } + } + /* + * Ensure that all required items are present + */ + for (i=0; job_items[i].name; i++) { + if (job_items[i].flags & ITEM_REQUIRED) { + if (!bit_is_set(i, job->hdr.item_present)) { + Jmsg(NULL, M_ERROR_TERM, 0, _("\"%s\" directive in Job \"%s\" resource is required, but not found.\n"), + job_items[i].name, job->name()); + OK = false; + } + } + /* If this triggers, take a look at lib/parse_conf.h */ + if (i >= MAX_RES_ITEMS) { + Emsg0(M_ERROR_TERM, 0, _("Too many items in Job resource\n")); + } + } + if (!job->storage && !job->pool->storage) { + Jmsg(NULL, M_FATAL, 0, _("No storage specified in Job \"%s\" nor in Pool.\n"), + job->name()); + OK = false; + } + } /* End loop over Job res */ + + UnlockRes(); + return OK; +} + +static void sendit(void *sock, const char *fmt, ...) +{ + char buf[3000]; + va_list arg_ptr; + + va_start(arg_ptr, fmt); + bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); + va_end(arg_ptr); + fputs(buf, stdout); + fflush(stdout); +} diff --git a/bacula/src/dird/bsr.c b/bacula/src/dird/bsr.c index 0530ed84e3..3af5afd03d 100644 --- a/bacula/src/dird/bsr.c +++ b/bacula/src/dird/bsr.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -17,14 +17,12 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- Bootstrap Record routines. * * BSR (bootstrap record) handling routines split from * ua_restore.c July MMIII * * Kern Sibbald, July MMII - * */ #include "bacula.h" @@ -43,17 +41,7 @@ RBSR_FINDEX *new_findex() return fi; } -/* Free all BSR FileIndex entries */ -static void free_findex(RBSR_FINDEX *fi) -{ - RBSR_FINDEX *next; - for ( ; fi; fi=next) { - next = fi->next; - free(fi); - } -} - -/* +/* * Get storage device name from Storage resource */ static bool get_storage_device(char *device, char *storage) @@ -62,7 +50,7 @@ static bool get_storage_device(char *device, char *storage) if (storage[0] == 0) { return false; } - store = (STORE *)GetResWithName(R_STORAGE, storage); + store = (STORE *)GetResWithName(R_STORAGE, storage); if (!store) { return false; } @@ -84,17 +72,48 @@ static bool get_storage_device(char *device, char *storage) * We are called here once for each JobMedia record * for each Volume. */ -static uint32_t write_findex(RBSR_FINDEX *fi, +static uint32_t write_findex(rblist *fi_list, int32_t FirstIndex, int32_t LastIndex, FILE *fd) { + RBSR_FINDEX *fi; uint32_t count = 0; - for ( ; fi; fi=fi->next) { + + fi = (RBSR_FINDEX *) fi_list->first(); + while (fi) { int32_t findex, findex2; - if ((fi->findex >= FirstIndex && fi->findex <= LastIndex) || - (fi->findex2 >= FirstIndex && fi->findex2 <= LastIndex) || - (fi->findex < FirstIndex && fi->findex2 > LastIndex)) { - findex = fi->findex < FirstIndex ? FirstIndex : fi->findex; - findex2 = fi->findex2 > LastIndex ? LastIndex : fi->findex2; + + /* fi points to the first item of the list, or the next item that is not + * contigous to the previous group + */ + findex = fi->findex; + findex2 = fi->findex2; + + /* Sometime (with the restore command for example), the fi_list can + * contain false gaps (1-10, 11-11, 12-20 instead of 1-20). The for loop + * is here to merge blocks and reduce the bsr output. The next while(fi) + * iteration will use the next_fi that points to the last merged element. + */ + RBSR_FINDEX *next_fi; + for (next_fi = (RBSR_FINDEX*) fi_list->next(fi); + next_fi && next_fi->findex == (findex2+1); + next_fi = (RBSR_FINDEX *) fi_list->next(next_fi)) + { + findex2 = next_fi->findex2; + } + + /* next_fi points after the current block (or to the end of the list), so + * the next while() iteration will use the next value + */ + fi = next_fi; + + /* We look if the current FI block match the volume information */ + if ((findex >= FirstIndex && findex <= LastIndex) || + (findex2 >= FirstIndex && findex2 <= LastIndex) || + (findex < FirstIndex && findex2 > LastIndex)) { + + findex = findex < FirstIndex ? FirstIndex : findex; + findex2 = findex2 > LastIndex ? LastIndex : findex2; + if (findex == findex2) { fprintf(fd, "FileIndex=%d\n", findex); count++; @@ -104,6 +123,7 @@ static uint32_t write_findex(RBSR_FINDEX *fi, } } } + return count; } @@ -111,10 +131,11 @@ static uint32_t write_findex(RBSR_FINDEX *fi, * Find out if Volume defined with FirstIndex and LastIndex * falls within the range of selected files in the bsr. */ -static bool is_volume_selected(RBSR_FINDEX *fi, +static bool is_volume_selected(rblist *fi_list, int32_t FirstIndex, int32_t LastIndex) { - for ( ; fi; fi=fi->next) { + RBSR_FINDEX *fi; + foreach_rblist(fi, fi_list) { if ((fi->findex >= FirstIndex && fi->findex <= LastIndex) || (fi->findex2 >= FirstIndex && fi->findex2 <= LastIndex) || (fi->findex < FirstIndex && fi->findex2 > LastIndex)) { @@ -128,35 +149,40 @@ static bool is_volume_selected(RBSR_FINDEX *fi, /* Create a new bootstrap record */ RBSR *new_bsr() { + RBSR_FINDEX *fi=NULL; RBSR *bsr = (RBSR *)bmalloc(sizeof(RBSR)); memset(bsr, 0, sizeof(RBSR)); + bsr->fi_list = New(rblist(fi, &fi->link)); return bsr; } /* Free the entire BSR */ -void free_bsr(RBSR *bsr) +void free_bsr(rblist *bsr_list) { - RBSR *next; - for ( ; bsr; bsr=next) { - free_findex(bsr->fi); + RBSR *bsr; + foreach_rblist(bsr, bsr_list) { + delete bsr->fi_list; if (bsr->VolParams) { free(bsr->VolParams); } if (bsr->fileregex) { free(bsr->fileregex); } - next = bsr->next; - free(bsr); + if (bsr->m_fi) { + free(bsr->m_fi); + } } + delete bsr_list; } /* * Complete the BSR by filling in the VolumeName and * VolSessionId and VolSessionTime using the JobId */ -bool complete_bsr(UAContext *ua, RBSR *bsr) +bool complete_bsr(UAContext *ua, rblist *bsr_list) { - for ( ; bsr; bsr=bsr->next) { + RBSR *bsr; + foreach_rblist(bsr, bsr_list) { JOB_DBR jr; memset(&jr, 0, sizeof(jr)); jr.JobId = bsr->JobId; @@ -191,7 +217,7 @@ static void make_unique_restore_filename(UAContext *ua, POOL_MEM &fname) JCR *jcr = ua->jcr; int i = find_arg_with_value(ua, "bootstrap"); if (i >= 0) { - Mmsg(fname, "%s", ua->argv[i]); + Mmsg(fname, "%s", ua->argv[i]); jcr->unlink_bsr = false; } else { P(mutex); @@ -217,7 +243,7 @@ uint32_t write_bsr_file(UAContext *ua, RESTORE_CTX &rx) bool err; make_unique_restore_filename(ua, fname); - fd = fopen(fname.c_str(), "w+b"); + fd = bfopen(fname.c_str(), "w+b"); if (!fd) { berrno be; ua->error_msg(_("Unable to create bootstrap file %s. ERR=%s\n"), @@ -238,9 +264,7 @@ uint32_t write_bsr_file(UAContext *ua, RESTORE_CTX &rx) goto bail_out; } - ua->send_msg(_("Bootstrap records written to %s\n"), fname.c_str()); - - if (debug_level >= 10) { + if (chk_dbglvl(10)) { print_bsr(ua, rx); } @@ -255,7 +279,7 @@ static void display_vol_info(UAContext *ua, RESTORE_CTX &rx, JobId_t JobId) char online; RBSR *bsr; - for (bsr=rx.bsr; bsr; bsr=bsr->next) { + foreach_rblist(bsr, rx.bsr_list) { if (JobId && JobId != bsr->JobId) { continue; } @@ -270,7 +294,7 @@ static void display_vol_info(UAContext *ua, RESTORE_CTX &rx, JobId_t JobId) } else { online = ' '; } - Mmsg(volmsg, "%c%-25s %-25s %-25s", + Mmsg(volmsg, "%c%-25s %-25s %-25s", online, bsr->VolParams[i].VolumeName, bsr->VolParams[i].Storage, Device); add_prompt(ua, volmsg.c_str()); @@ -319,7 +343,7 @@ void display_bsr_info(UAContext *ua, RESTORE_CTX &rx) /* * Write bsr data for a single bsr record */ -static uint32_t write_bsr_item(RBSR *bsr, UAContext *ua, +static uint32_t write_bsr_item(RBSR *bsr, UAContext *ua, RESTORE_CTX &rx, FILE *fd, bool &first, uint32_t &LastIndex) { char ed1[50], ed2[50]; @@ -332,7 +356,7 @@ static uint32_t write_bsr_item(RBSR *bsr, UAContext *ua, * VolCount is the number of JobMedia records. */ for (int i=0; i < bsr->VolCount; i++) { - if (!is_volume_selected(bsr->fi, bsr->VolParams[i].FirstIndex, + if (!is_volume_selected(bsr->fi_list, bsr->VolParams[i].FirstIndex, bsr->VolParams[i].LastIndex)) { bsr->VolParams[i].VolumeName[0] = 0; /* zap VolumeName */ continue; @@ -357,10 +381,10 @@ static uint32_t write_bsr_item(RBSR *bsr, UAContext *ua, fprintf(fd, "VolSessionTime=%u\n", bsr->VolSessionTime); fprintf(fd, "VolAddr=%s-%s\n", edit_uint64(bsr->VolParams[i].StartAddr, ed1), edit_uint64(bsr->VolParams[i].EndAddr, ed2)); -// Dmsg2(100, "bsr VolParam FI=%u LI=%u\n", -// bsr->VolParams[i].FirstIndex, bsr->VolParams[i].LastIndex); + Dmsg2(100, "bsr VolParam FI=%u LI=%u\n", + bsr->VolParams[i].FirstIndex, bsr->VolParams[i].LastIndex); - count = write_findex(bsr->fi, bsr->VolParams[i].FirstIndex, + count = write_findex(bsr->fi_list, bsr->VolParams[i].FirstIndex, bsr->VolParams[i].LastIndex, fd); if (count) { fprintf(fd, "Count=%u\n", count); @@ -384,7 +408,7 @@ static uint32_t write_bsr_item(RBSR *bsr, UAContext *ua, * Here we actually write out the details of the bsr file. * Note, there is one bsr for each JobId, but the bsr may * have multiple volumes, which have been entered in the - * order they were written. + * order they were written. * The bsrs must be written out in the order the JobIds * are found in the jobid list. */ @@ -397,13 +421,13 @@ static uint32_t write_bsr(UAContext *ua, RESTORE_CTX &rx, FILE *fd) JobId_t JobId; RBSR *bsr; if (*rx.JobIds == 0) { - for (bsr=rx.bsr; bsr; bsr=bsr->next) { + foreach_rblist(bsr, rx.bsr_list) { total_count += write_bsr_item(bsr, ua, rx, fd, first, LastIndex); } return total_count; } for (p=rx.JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) { - for (bsr=rx.bsr; bsr; bsr=bsr->next) { + foreach_rblist(bsr, rx.bsr_list) { if (JobId == bsr->JobId) { total_count += write_bsr_item(bsr, ua, rx, fd, first, LastIndex); } @@ -417,105 +441,123 @@ void print_bsr(UAContext *ua, RESTORE_CTX &rx) write_bsr(ua, rx, stdout); } +static int search_rbsr(void *elt1, void *elt2) +{ + RBSR *bsr1 = (RBSR *)elt1; + RBSR *bsr = (RBSR *)elt2; + + /* We might replace by a simple JobId - JobId */ + if (bsr->JobId == bsr1->JobId) { + return 0; + + } else if (bsr->JobId < bsr1->JobId) { + return 1; + } + + return -1; +} +static int search_fi(void *elt1, void *elt2) +{ + RBSR_FINDEX *f1 = (RBSR_FINDEX *) elt1; + RBSR_FINDEX *f2 = (RBSR_FINDEX *) elt2; + if (f1->findex == (f2->findex - 1)) { + return 0; + + } else if (f1->findex2 == (f2->findex2 + 1)) { + return 0; + + } else if (f1->findex >= f2->findex && f1->findex2 <= f2->findex2) { + return 0; + } + + return (f1->findex > f2->findex) ? 1 : -1; +} + +rblist *create_bsr_list(uint32_t JobId, int findex, int findex2) +{ + RBSR *bsr = NULL; + RBSR_FINDEX *fi = NULL; + rblist *bsr_list = New(rblist(bsr, &bsr->link)); + + bsr = new_bsr(); + bsr->JobId = JobId; + + bsr_list->insert(bsr, search_rbsr); + + fi = new_findex(); + fi->findex = findex; + fi->findex2 = findex2; + + bsr->fi_list->insert(fi, search_fi); + + return bsr_list; +} /* * Add a FileIndex to the list of BootStrap records. * Here we are only dealing with JobId's and the FileIndexes * associated with those JobIds. * We expect that JobId, FileIndex are sorted ascending. + * + * When doing restore from tree, FileIndex are not sorted, so it can + * create gaps. */ -void add_findex(RBSR *bsr, uint32_t JobId, int32_t findex) +void add_findex(rblist *bsr_list, uint32_t JobId, int32_t findex) { - RBSR *nbsr; - RBSR_FINDEX *fi, *lfi; + RBSR *bsr, bsr2; + RBSR_FINDEX *fi, *nfi; if (findex == 0) { return; /* probably a dummy directory */ } - if (bsr->fi == NULL) { /* if no FI add one */ - /* This is the first FileIndex item in the chain */ - bsr->fi = new_findex(); - bsr->JobId = JobId; - bsr->fi->findex = findex; - bsr->fi->findex2 = findex; - return; - } + bsr2.JobId = JobId; /* Walk down list of bsrs until we find the JobId */ - if (bsr->JobId != JobId) { - for (nbsr=bsr->next; nbsr; nbsr=nbsr->next) { - if (nbsr->JobId == JobId) { - bsr = nbsr; - break; - } - } + bsr = (RBSR *)bsr_list->search(&bsr2, search_rbsr); - if (!nbsr) { /* Must add new JobId */ - /* Add new JobId at end of chain */ - for (nbsr=bsr; nbsr->next; nbsr=nbsr->next) - { } - nbsr->next = new_bsr(); - nbsr->next->JobId = JobId; - nbsr->next->fi = new_findex(); - nbsr->next->fi->findex = findex; - nbsr->next->fi->findex2 = findex; - return; - } + /* The list is empty, or the JobId is not already in, + * Must add new JobId + */ + if (!bsr) { + bsr = new_bsr(); + bsr->JobId = JobId; + bsr_list->insert(bsr, search_rbsr); } - /* - * At this point, bsr points to bsr containing this JobId, - * and we are sure that there is at least one fi record. - */ - lfi = fi = bsr->fi; - /* Check if this findex is smaller than first item */ - if (findex < fi->findex) { - if ((findex+1) == fi->findex) { - fi->findex = findex; /* extend down */ - return; - } - fi = new_findex(); /* yes, insert before first item */ - fi->findex = findex; - fi->findex2 = findex; - fi->next = lfi; - bsr->fi = fi; - return; + if (bsr->m_fi) { + fi = bsr->m_fi; + + } else { + fi = bsr->m_fi = new_findex(); } - /* Walk down fi chain and find where to insert insert new FileIndex */ - for ( ; fi; fi=fi->next) { - if (findex == (fi->findex2 + 1)) { /* extend up */ - RBSR_FINDEX *nfi; - fi->findex2 = findex; - /* - * If the following record contains one higher, merge its - * file index by extending it up. - */ - if (fi->next && ((findex+1) == fi->next->findex)) { - nfi = fi->next; - fi->findex2 = nfi->findex2; - fi->next = nfi->next; - free(nfi); - } - return; - } - if (findex < fi->findex) { /* add before */ - if ((findex+1) == fi->findex) { - fi->findex = findex; - return; - } - break; + + fi->findex = findex; + fi->findex2 = findex; + + Dmsg1(1000, "Trying to insert %ld\n", findex); + /* try to insert our fi */ + nfi = (RBSR_FINDEX*) bsr->fi_list->insert((void *)fi, search_fi); + + /* We found an existing one, extend it */ + if (nfi != fi) { + if (findex == (nfi->findex2 + 1)) { + Dmsg2(1000, "Extend %ld-%ld\n", nfi->findex, findex); + nfi->findex2 = findex; + + } else if (findex == (nfi->findex - 1)) { + Dmsg2(1000, "Extend %ld-%ld\n", findex, nfi->findex2); + nfi->findex = findex; + + } else { + Dmsg2(1000, "Found the same values? %ld-%ld\n", nfi->findex, nfi->findex2); } - lfi = fi; + + } else { + Dmsg2(1000, "Inserted %ld-%ld\n", fi->findex, fi->findex2); + bsr->m_fi = NULL; /* comsumed */ } - /* Add to last place found */ - fi = new_findex(); - fi->findex = findex; - fi->findex2 = findex; - fi->next = lfi->next; - lfi->next = fi; - return; } /* @@ -523,54 +565,38 @@ void add_findex(RBSR *bsr, uint32_t JobId, int32_t findex) * Here we are only dealing with JobId's and the FileIndexes * associated with those JobIds. */ -void add_findex_all(RBSR *bsr, uint32_t JobId) +void add_findex_all(rblist *bsr_list, uint32_t JobId, const char *fileregex) { - RBSR *nbsr; + RBSR *bsr, bsr2; RBSR_FINDEX *fi; - if (bsr->fi == NULL) { /* if no FI add one */ - /* This is the first FileIndex item in the chain */ - bsr->fi = new_findex(); - bsr->JobId = JobId; - bsr->fi->findex = 1; - bsr->fi->findex2 = INT32_MAX; - return; - } + bsr2.JobId = JobId; /* Walk down list of bsrs until we find the JobId */ - if (bsr->JobId != JobId) { - for (nbsr=bsr->next; nbsr; nbsr=nbsr->next) { - if (nbsr->JobId == JobId) { - bsr = nbsr; - break; - } - } + bsr = (RBSR *)bsr_list->search(&bsr2, search_rbsr); - if (!nbsr) { /* Must add new JobId */ - /* Add new JobId at end of chain */ - for (nbsr=bsr; nbsr->next; nbsr=nbsr->next) - { } + if (!bsr) { /* Must add new JobId */ + fi = new_findex(); + fi->findex = 1; + fi->findex2 = INT32_MAX; - nbsr->next = new_bsr(); - nbsr->next->JobId = JobId; + bsr = new_bsr(); + bsr->JobId = JobId; + bsr->fi_list->insert(fi, search_fi); + bsr_list->insert(bsr, search_rbsr); + if (fileregex) { /* If we use regexp to restore, set it for each jobid */ - if (bsr->fileregex) { - nbsr->next->fileregex = bstrdup(bsr->fileregex); - } - - nbsr->next->fi = new_findex(); - nbsr->next->fi->findex = 1; - nbsr->next->fi->findex2 = INT32_MAX; - return; + bsr->fileregex = bstrdup(fileregex); } + return; } /* * At this point, bsr points to bsr containing this JobId, - * and we are sure that there is at least one fi record. */ - fi = bsr->fi; + fi = new_findex(); fi->findex = 1; fi->findex2 = INT32_MAX; + bsr->fi_list->insert(fi, search_fi); return; } diff --git a/bacula/src/dird/bsr.h b/bacula/src/dird/bsr.h index 216e16f078..0e4ae358a8 100644 --- a/bacula/src/dird/bsr.h +++ b/bacula/src/dird/bsr.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2002-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -18,21 +17,19 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bootstrap Record header file * * BSR (bootstrap record) handling routines split from * ua_restore.c July MMIII * * Kern Sibbald, July MMII - * */ /* FileIndex entry in restore bootstrap record */ struct RBSR_FINDEX { - RBSR_FINDEX *next; + rblink link; int32_t findex; int32_t findex2; }; @@ -46,12 +43,15 @@ struct RBSR_FINDEX { * on which the Job is stored to the BSR. */ struct RBSR { - RBSR *next; /* next JobId */ + rblink link; JobId_t JobId; /* JobId this bsr */ uint32_t VolSessionId; uint32_t VolSessionTime; int VolCount; /* Volume parameter count */ VOL_PARAMS *VolParams; /* Volume, start/end file/blocks */ - RBSR_FINDEX *fi; /* File indexes this JobId */ - char *fileregex; /* Only restore files matching regex */ + rblist *fi_list; /* File indexes this JobId */ + char *fileregex; /* Only restore files matching regex */ + + /* If we extend an existing fi, keep the memory for the next insert */ + RBSR_FINDEX *m_fi; }; diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index ed0a736784..9c9d1f3d84 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- catreq.c -- handles the message channel * catalog request from the Storage daemon. * @@ -27,7 +26,6 @@ * * Basic tasks done here: * Handle Catalog services. - * */ #include "bacula.h" @@ -40,17 +38,18 @@ */ /* Requests from the Storage daemon */ -static char Find_media[] = "CatReq Job=%127s FindMedia=%d pool_name=%127s media_type=%127s vol_type=%d\n"; -static char Get_Vol_Info[] = "CatReq Job=%127s GetVolInfo VolName=%127s write=%d\n"; +static char Find_media[] = "CatReq JobId=%ld FindMedia=%d pool_name=%127s media_type=%127s vol_type=%d\n"; +static char Get_Vol_Info[] = "CatReq JobId=%ld GetVolInfo VolName=%127s write=%d\n"; -static char Update_media[] = "CatReq Job=%127s UpdateMedia VolName=%s" +static char Update_media[] = "CatReq JobId=%ld UpdateMedia VolName=%s" " VolJobs=%u VolFiles=%u VolBlocks=%u VolBytes=%lld VolABytes=%lld" " VolHoleBytes=%lld VolHoles=%u VolMounts=%u" " VolErrors=%u VolWrites=%lld MaxVolBytes=%lld EndTime=%lld VolStatus=%10s" " Slot=%d relabel=%d InChanger=%d VolReadTime=%lld VolWriteTime=%lld" - " VolFirstWritten=%lld VolType=%u\n"; + " VolFirstWritten=%lld VolType=%u VolParts=%d VolCloudParts=%d" + " LastPartBytes=%lld Enabled=%d\n"; -static char Create_jobmedia[] = "CatReq Job=%127s CreateJobMedia\n"; +static char Create_jobmedia[] = "CatReq JobId=%ld CreateJobMedia\n"; /* Responses sent to Storage daemon */ static char OK_media[] = "1000 OK VolName=%s VolJobs=%u VolFiles=%u" @@ -59,11 +58,25 @@ static char OK_media[] = "1000 OK VolName=%s VolJobs=%u VolFiles=%u" " MaxVolBytes=%s VolCapacityBytes=%s VolStatus=%s Slot=%d" " MaxVolJobs=%u MaxVolFiles=%u InChanger=%d VolReadTime=%s" " VolWriteTime=%s EndFile=%u EndBlock=%u VolType=%u LabelType=%d" - " MediaId=%s ScratchPoolId=%s\n"; + " MediaId=%s ScratchPoolId=%s VolParts=%d VolCloudParts=%d" + " LastPartBytes=%lld Enabled=%d\n"; static char OK_create[] = "1000 OK CreateJobMedia\n"; +void remove_dummy_jobmedia_records(JCR *jcr) +{ + if (jcr->dummy_jobmedia) { + char ec1[30]; + POOL_MEM buf; + Mmsg(buf, "DELETE FROM JobMedia WHERE JobId=%s AND FirstIndex=0 AND LastIndex=0", + edit_int64(jcr->JobId, ec1)); + Dmsg1(150, "Delete dummy: %s\n", buf.c_str()); + db_sql_query(jcr->db, buf.c_str(), NULL, NULL); + jcr->dummy_jobmedia = false; + } +} + static int send_volume_info_to_storage_daemon(JCR *jcr, BSOCK *sd, MEDIA_DBR *mr) { int stat; @@ -89,7 +102,11 @@ static int send_volume_info_to_storage_daemon(JCR *jcr, BSOCK *sd, MEDIA_DBR *mr mr->VolType, mr->LabelType, edit_uint64(mr->MediaId, ed9), - edit_uint64(mr->ScratchPoolId, ed10)); + edit_uint64(mr->ScratchPoolId, ed10), + mr->VolParts, + mr->VolCloudParts, + mr->LastPartBytes, + mr->Enabled); unbash_spaces(mr->VolumeName); Dmsg2(100, "Vol Info for %s: %s", jcr->Job, sd->msg); return stat; @@ -102,7 +119,6 @@ void catalog_request(JCR *jcr, BSOCK *bs) { MEDIA_DBR mr, sdmr; JOBMEDIA_DBR jm; - char Job[MAX_NAME_LENGTH]; char pool_name[MAX_NAME_LENGTH]; int index, ok, label, writing; POOLMEM *omsg; @@ -111,6 +127,8 @@ void catalog_request(JCR *jcr, BSOCK *bs) utime_t VolFirstWritten; utime_t VolLastWritten; int n; + int Enabled; + JobId_t JobId = 0; memset(&sdmr, 0, sizeof(sdmr)); memset(&jm, 0, sizeof(jm)); @@ -131,7 +149,7 @@ void catalog_request(JCR *jcr, BSOCK *bs) /* * Find next appendable medium for SD */ - n = sscanf(bs->msg, Find_media, &Job, &index, &pool_name, &mr.MediaType, &mr.VolType); + n = sscanf(bs->msg, Find_media, &JobId, &index, &pool_name, &mr.MediaType, &mr.VolType); if (n == 5) { memset(&pr, 0, sizeof(pr)); bstrncpy(pr.Name, pool_name, sizeof(pr.Name)); @@ -164,7 +182,7 @@ void catalog_request(JCR *jcr, BSOCK *bs) /* * Request to find specific Volume information */ - n = sscanf(bs->msg, Get_Vol_Info, &Job, &mr.VolumeName, &writing); + n = sscanf(bs->msg, Get_Vol_Info, &JobId, &mr.VolumeName, &writing); if (n == 3) { Dmsg1(100, "CatReq GetVolInfo Vol=%s\n", mr.VolumeName); /* @@ -225,14 +243,15 @@ void catalog_request(JCR *jcr, BSOCK *bs) * of a Storage daemon Job Session, when labeling/relabeling a * Volume, or when an EOF mark is written. */ - n = sscanf(bs->msg, Update_media, &Job, &sdmr.VolumeName, + n = sscanf(bs->msg, Update_media, &JobId, &sdmr.VolumeName, &sdmr.VolJobs, &sdmr.VolFiles, &sdmr.VolBlocks, &sdmr.VolBytes, &sdmr.VolABytes, &sdmr.VolHoleBytes, &sdmr.VolHoles, &sdmr.VolMounts, &sdmr.VolErrors, &sdmr.VolWrites, &sdmr.MaxVolBytes, &VolLastWritten, &sdmr.VolStatus, &sdmr.Slot, &label, &sdmr.InChanger, &sdmr.VolReadTime, &sdmr.VolWriteTime, &VolFirstWritten, - &sdmr.VolType); - if (n == 22) { + &sdmr.VolType, &sdmr.VolParts, &sdmr.VolCloudParts, + &sdmr.LastPartBytes, &Enabled); + if (n == 26) { db_lock(jcr->db); Dmsg3(400, "Update media %s oldStat=%s newStat=%s\n", sdmr.VolumeName, mr.VolStatus, sdmr.VolStatus); @@ -300,19 +319,23 @@ void catalog_request(JCR *jcr, BSOCK *bs) } /* Copy updated values to original media record */ - mr.VolJobs = sdmr.VolJobs; - mr.VolFiles = sdmr.VolFiles; - mr.VolBlocks = sdmr.VolBlocks; - mr.VolBytes = sdmr.VolBytes; - mr.VolABytes = sdmr.VolABytes; - mr.VolHoleBytes = sdmr.VolHoleBytes; - mr.VolHoles = sdmr.VolHoles; - mr.VolMounts = sdmr.VolMounts; - mr.VolErrors = sdmr.VolErrors; - mr.VolWrites = sdmr.VolWrites; - mr.Slot = sdmr.Slot; - mr.InChanger = sdmr.InChanger; - mr.VolType = sdmr.VolType; + mr.VolJobs = sdmr.VolJobs; + mr.VolFiles = sdmr.VolFiles; + mr.VolBlocks = sdmr.VolBlocks; + mr.VolBytes = sdmr.VolBytes; + mr.VolABytes = sdmr.VolABytes; + mr.VolHoleBytes = sdmr.VolHoleBytes; + mr.VolHoles = sdmr.VolHoles; + mr.VolMounts = sdmr.VolMounts; + mr.VolErrors = sdmr.VolErrors; + mr.VolWrites = sdmr.VolWrites; + mr.Slot = sdmr.Slot; + mr.InChanger = sdmr.InChanger; + mr.VolType = sdmr.VolType; + mr.VolParts = sdmr.VolParts; + mr.VolCloudParts = sdmr.VolCloudParts; + mr.LastPartBytes = sdmr.LastPartBytes; + mr.Enabled = Enabled; /* byte assignment */ bstrncpy(mr.VolStatus, sdmr.VolStatus, sizeof(mr.VolStatus)); if (sdmr.VolReadTime >= 0) { mr.VolReadTime = sdmr.VolReadTime; @@ -338,12 +361,12 @@ void catalog_request(JCR *jcr, BSOCK *bs) db_unlock(jcr->db); goto ok_out; } - Dmsg1(1000, "Tried update_media. fields wanted=20, got=%d\n", n); + Dmsg1(1000, "Tried update_media. fields wanted=25, got=%d\n", n); /* * Request to create a JobMedia record */ - if (sscanf(bs->msg, Create_jobmedia, &Job) == 1) { + if (sscanf(bs->msg, Create_jobmedia, &JobId) == 1) { if (jcr->wjcr) { jm.JobId = jcr->wjcr->JobId; } else { @@ -361,9 +384,12 @@ void catalog_request(JCR *jcr, BSOCK *bs) } if (ok) { jm.MediaId = MediaId; - Dmsg6(400, "create_jobmedia JobId=%d MediaId=%d SF=%d EF=%d FI=%d LI=%d\n", + Dmsg6(400, "create_jobmedia JobId=%ld MediaId=%lu SF=%lu EF=%lu FI=%lu LI=%lu\n", jm.JobId, jm.MediaId, jm.StartFile, jm.EndFile, jm.FirstIndex, jm.LastIndex); ok = db_create_jobmedia_record(jcr, jcr->db, &jm); + if (jm.FirstIndex == 0 && jm.LastIndex == 0) { + jcr->dummy_jobmedia = true; + } } } db_end_transaction(jcr, jcr->db); @@ -372,7 +398,7 @@ void catalog_request(JCR *jcr, BSOCK *bs) db_strerror(jcr->db)); db_unlock(jcr->db); bs->fsend(_("1992 Create JobMedia error\n")); - goto ok_out; + goto ok_out; } db_unlock(jcr->db); Dmsg0(400, "JobMedia record created\n"); @@ -539,7 +565,7 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) ro.FileIndex = FileIndex; if (jcr->wjcr) { ro.JobId = jcr->wjcr->JobId; - Dmsg1(100, "=== set JobId=%d\n", ar->JobId); + Dmsg1(100, "=== set JobId=%ld\n", ar->JobId); } else { ro.JobId = jcr->JobId; } @@ -570,7 +596,7 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) len = strlen(ro.object_name); ro.object = &ro.object_name[len+1]; /* point to object */ ro.object[ro.object_len] = 0; /* add zero for those who attempt printing */ - Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%d, obj_len=%d\nobj=\"%s\"\n", + Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%ld, obj_len=%d\nobj=\"%s\"\n", ro.object_name, ro.Stream, ro.FileType, ro.FileIndex, ro.JobId, ro.object_len, ro.object); /* Send it */ @@ -581,7 +607,7 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) } else if (crypto_digest_stream_type(Stream) != CRYPTO_DIGEST_NONE) { fname = p; if (ar->FileIndex != FileIndex) { - Jmsg3(jcr, M_WARNING, 0, _("%s not same File=%d as attributes=%d\n"), + Jmsg3(jcr, M_WARNING, 0, _("%s not same FileIndex=%d as attributes FI=%d\n"), stream_to_ascii(Stream), FileIndex, ar->FileIndex); } else { /* Update digest in catalog */ @@ -679,11 +705,12 @@ bool despool_attributes_from_file(JCR *jcr, const char *file) { bool ret=false; int32_t pktsiz; - size_t nbytes; + ssize_t nbytes; ssize_t size = 0; int32_t msglen; /* message length */ POOLMEM *msg = get_pool_memory(PM_MESSAGE); FILE *spool_fd=NULL; + int32_t recnum = 0; Dmsg1(100, "Begin despool_attributes_from_file\n", file); @@ -691,7 +718,8 @@ bool despool_attributes_from_file(JCR *jcr, const char *file) goto bail_out; /* user disabled cataloging */ } - spool_fd = fopen(file, "rb"); + spool_fd = bfopen(file, "rb"); + //Dmsg1(000, "Open attr read file=%s\n", file); if (!spool_fd) { Dmsg0(100, "cancel despool_attributes_from_file\n"); /* send an error message */ @@ -701,20 +729,44 @@ bool despool_attributes_from_file(JCR *jcr, const char *file) posix_fadvise(fileno(spool_fd), 0, 0, POSIX_FADV_WILLNEED); #endif - while (fread((char *)&pktsiz, 1, sizeof(int32_t), spool_fd) == - sizeof(int32_t)) { + /* + * We read the attributes file or stream from the SD. It should + * be in the following format: + * + * 1. 4 bytes representing the record length + * 2. An attribute string starting with: UpdCat Job=nnn FileAttributes ... + */ + for ( ;; ) { + nbytes = fread((char *)&pktsiz, 1, sizeof(int32_t), spool_fd); + if (nbytes == 0) { /* EOF */ + break; + } + if (nbytes != sizeof(int32_t)) { + Dmsg2(000, "Error: attr read status=%lld addr=%lld\n", nbytes, ftello(spool_fd)); + break; + } size += sizeof(int32_t); msglen = ntohl(pktsiz); + if (msglen > 10000000) { + Qmsg1(jcr, M_FATAL, 0, _("fread attr spool error. Wanted %ld bytes, maximum permitted 10000000 bytes\n"), msglen); + goto bail_out; + } if (msglen > 0) { - if (msglen > (int32_t) sizeof_pool_memory(msg)) { + if (msglen > (int32_t)sizeof_pool_memory(msg)) { msg = realloc_pool_memory(msg, msglen + 1); } nbytes = fread(msg, 1, msglen, spool_fd); - if (nbytes != (size_t) msglen) { + recnum++; + if (nbytes > 0 && strncmp(msg, "UpdCat Job", 10) != 0) { + Dmsg3(000, "Error: recnum=%ld nbytes=%lld msg=%s\n", recnum, nbytes, msg); + } + if (nbytes != (ssize_t)msglen) { berrno be; - Dmsg2(400, "nbytes=%d msglen=%d\n", nbytes, msglen); - Qmsg1(jcr, M_FATAL, 0, _("fread attr spool error. ERR=%s\n"), - be.bstrerror()); + boffset_t size; + size = ftello(spool_fd); + Dmsg4(000, "Error at size=%lld record %ld: got nbytes=%lld, want msglen=%ld\n", size, recnum, (int32_t)nbytes, msglen); + Qmsg3(jcr, M_FATAL, 0, _("fread attr spool error. Wanted %ld bytes but got %lld ERR=%s\n"), + msglen, nbytes, be.bstrerror()); goto bail_out; } size += nbytes; @@ -737,6 +789,7 @@ bool despool_attributes_from_file(JCR *jcr, const char *file) bail_out: if (spool_fd) { + //Dmsg1(000, "Close attr read file=%s\n", file); fclose(spool_fd); } diff --git a/bacula/src/dird/dir_plugins.h b/bacula/src/dird/dir_plugins.h index 0fa7f721d7..45dab8837d 100644 --- a/bacula/src/dird/dir_plugins.h +++ b/bacula/src/dird/dir_plugins.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2007-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Interface definition for Bacula Plugins * * Kern Sibbald, October 2007 - * */ #ifndef __DIR_PLUGINS_H diff --git a/bacula/src/dird/dird.c b/bacula/src/dird/dird.c index f74097f840..1f85ac7247 100644 --- a/bacula/src/dird/dird.c +++ b/bacula/src/dird/dird.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,17 +11,15 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director daemon -- this is the main program * * Kern Sibbald, March MM - * */ #include "bacula.h" @@ -43,6 +41,7 @@ int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); void terminate_dird(int sig); static bool check_resources(); static void cleanup_old_files(); +static void resize_reload(int nb); /* Exported subroutines */ extern "C" void reload_config(int sig); @@ -64,7 +63,7 @@ void init_device_resources(); static char *runjob = NULL; -static bool background = true; +static bool foreground = false; static void init_reload(void); static CONFIG *config; static bool test_config = false; @@ -77,7 +76,14 @@ char *configfile = NULL; void *start_heap; utime_t last_reload_time = 0; + /* Globals Imported */ +extern dlist client_globals; +extern dlist store_globals; +extern dlist job_globals; +extern dlist sched_globals; +extern dlist *daemon_msg_queue; +extern pthread_mutex_t daemon_msg_queue_mutex; extern RES_ITEM job_items[]; #if defined(_MSC_VER) extern "C" { // work around visual compiler mangling variables @@ -117,7 +123,7 @@ static void usage() { fprintf(stderr, _( PROG_COPYRIGHT - "\nVersion: %s (%s)\n\n" + "\n%sVersion: %s (%s)\n\n" "Usage: bacula-dir [-f -s] [-c config_file] [-d debug_level] [config_file]\n" " -c set configuration file to file\n" " -d [,] set debug level to , debug tags to \n" @@ -132,7 +138,7 @@ static void usage() " -u userid\n" " -v verbose user messages\n" " -? print this message.\n" - "\n"), 2000, VERSION, BDATE); + "\n"), 2000, "", VERSION, BDATE); exit(1); } @@ -158,6 +164,10 @@ static void dir_debug_print(JCR *jcr, FILE *fp) #define main BaculaMain #endif +/* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ +extern void (*MA1512_reload_job_end_cb)(JCR *,void *); +static void reload_job_end_cb(JCR *jcr, void *ctx); + int main (int argc, char *argv[]) { int ch; @@ -165,6 +175,10 @@ int main (int argc, char *argv[]) bool no_signals = false; char *uid = NULL; char *gid = NULL; + MQUEUE_ITEM *item = NULL; + + /* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ + MA1512_reload_job_end_cb = reload_job_end_cb; start_heap = sbrk(0); setlocale(LC_ALL, ""); @@ -176,7 +190,8 @@ int main (int argc, char *argv[]) init_msg(NULL, NULL); /* initialize message handler */ init_reload(); daemon_start_time = time(NULL); - + /* Setup daemon message queue */ + daemon_msg_queue = New(dlist(item, &item->link)); console_command = run_console_command; while ((ch = getopt(argc, argv, "c:d:fg:mr:stu:v?T")) != -1) { @@ -213,7 +228,7 @@ int main (int argc, char *argv[]) break; case 'f': /* run in foreground */ - background = false; + foreground = true; break; case 'g': /* set group id */ @@ -258,10 +273,6 @@ int main (int argc, char *argv[]) argc -= optind; argv += optind; - if (!no_signals) { - init_signals(terminate_dird); - } - if (argc) { if (configfile != NULL) { free(configfile); @@ -274,19 +285,20 @@ int main (int argc, char *argv[]) usage(); } - if (!test_config) { /* we don't need to do this block in test mode */ - if (background) { - daemon_start(); - init_stack_dump(); /* grab new pid */ - } + if (!foreground && !test_config) { + daemon_start(); + init_stack_dump(); /* grab new pid */ + } + + if (!no_signals) { + init_signals(terminate_dird); } if (configfile == NULL) { configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); - + config = New(CONFIG()); parse_dir_config(config, configfile, M_ERROR_TERM); if (init_crypto() != 0) { @@ -336,6 +348,8 @@ int main (int argc, char *argv[]) FDConnectTimeout = (int)director->FDConnectTimeout; SDConnectTimeout = (int)director->SDConnectTimeout; + resize_reload(director->MaxReload); + #if !defined(HAVE_WIN32) signal(SIGHUP, reload_config); #endif @@ -374,17 +388,32 @@ int main (int argc, char *argv[]) struct RELOAD_TABLE { int job_count; - RES **res_table; + RES_HEAD **res_head; }; -static const int max_reloads = 50; -static RELOAD_TABLE reload_table[max_reloads]; +static int max_reloads = 32; +static RELOAD_TABLE *reload_table=NULL; + +static void resize_reload(int nb) +{ + if (nb <= max_reloads) { + return; + } + + reload_table = (RELOAD_TABLE*)realloc(reload_table, nb * sizeof(RELOAD_TABLE)); + for (int i=max_reloads; i < nb ; i++) { + reload_table[i].job_count = 0; + reload_table[i].res_head = NULL; + } + max_reloads = nb; +} static void init_reload(void) { + reload_table = (RELOAD_TABLE*)malloc(max_reloads * sizeof(RELOAD_TABLE)); for (int i=0; i < max_reloads; i++) { reload_table[i].job_count = 0; - reload_table[i].res_table = NULL; + reload_table[i].res_head = NULL; } } @@ -394,19 +423,31 @@ static void init_reload(void) */ static void free_saved_resources(int table) { + RES *next, *res; int num = r_last - r_first + 1; - RES **res_tab = reload_table[table].res_table; - if (!res_tab) { + RES_HEAD **res_tab = reload_table[table].res_head; + + if (res_tab == NULL) { Dmsg1(100, "res_tab for table %d already released.\n", table); return; } Dmsg1(100, "Freeing resources for table %d\n", table); for (int j=0; jfirst; + for ( ; next; ) { + res = next; + next = res->res_next; + free_resource(res, r_first + j); + } + free(res_tab[j]->res_list); + free(res_tab[j]); + res_tab[j] = NULL; + } } free(res_tab); reload_table[table].job_count = 0; - reload_table[table].res_table = NULL; + reload_table[table].res_head = NULL; } /* @@ -433,7 +474,7 @@ static int find_free_reload_table_entry() { int table = -1; for (int i=0; i < max_reloads; i++) { - if (reload_table[i].res_table == NULL) { + if (reload_table[i].res_head == NULL) { table = i; break; } @@ -441,6 +482,8 @@ static int find_free_reload_table_entry() return table; } +static pthread_mutex_t reload_mutex = PTHREAD_MUTEX_INITIALIZER; + /* * If we get here, we have received a SIGHUP, which means to * reread our configuration file. @@ -470,12 +513,27 @@ void reload_config(int sig) JCR *jcr; int njobs = 0; /* number of running jobs */ int table, rtable; - bool ok; - - if (already_here) { - abort(); /* Oops, recursion -> die */ - } - already_here = true; + bool ok=false; + int tries=0; + + /* Wait to do the reload */ + do { + P(reload_mutex); + if (already_here) { + V(reload_mutex); + if (tries++ > 10) { + Jmsg(NULL, M_INFO, 0, _("Already doing a reload request, " + "request ignored.\n")); + return; + } + Dmsg0(10, "Already doing a reload request, waiting a bit\n"); + bmicrosleep(1, 0); + } else { + already_here = true; + V(reload_mutex); + ok = true; + } + } while (!ok); #if !defined(HAVE_WIN32) sigemptyset(&set); @@ -488,18 +546,25 @@ void reload_config(int sig) table = find_free_reload_table_entry(); if (table < 0) { - Jmsg(NULL, M_ERROR, 0, _("Too many open reload requests. Request ignored.\n")); + Jmsg(NULL, M_ERROR, 0, _("Too many open reload requests. " + "Request ignored.\n")); goto bail_out; } Dmsg1(100, "Reload_config njobs=%d\n", njobs); - reload_table[table].res_table = config->save_resources(); + /* Save current res_head */ + reload_table[table].res_head = res_head; Dmsg1(100, "Saved old config in table %d\n", table); + /* Create a new res_head and parse into it */ ok = parse_dir_config(config, configfile, M_ERROR); Dmsg0(100, "Reloaded config file\n"); if (!ok || !check_resources() || !check_catalog(UPDATE_CATALOG)) { + /* + * We got an error, save broken point, restore old one, + * then release everything from broken pointer. + */ rtable = find_free_reload_table_entry(); /* save new, bad table */ if (rtable < 0) { Jmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); @@ -508,14 +573,12 @@ void reload_config(int sig) Jmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); Jmsg(NULL, M_ERROR, 0, _("Resetting previous configuration.\n")); } - reload_table[rtable].res_table = config->save_resources(); - /* Now restore old resource values */ - int num = r_last - r_first + 1; - RES **res_tab = reload_table[table].res_table; - for (int i=0; iname); + if (!client) { + Jmsg(NULL, M_INFO, 0, _("Client=%s not found. Assuming it was removed!!!\n"), cg->name); + } else { + client->globals = cg; /* Set globals pointer */ + } + } + STORE_GLOBALS *sg; + foreach_dlist(sg, &store_globals) { + STORE *store; + store = GetStoreResWithName(sg->name); + if (!store) { + Jmsg(NULL, M_INFO, 0, _("Storage=%s not found. Assuming it was removed!!!\n"), sg->name); + } else { + store->globals = sg; /* set globals pointer */ + Dmsg2(200, "Reload found numConcurrent=%ld for Store %s\n", + sg->NumConcurrentJobs, sg->name); + } + } + JOB_GLOBALS *jg; + foreach_dlist(jg, &job_globals) { + JOB *job; + job = GetJobResWithName(jg->name); + if (!job) { + Jmsg(NULL, M_INFO, 0, _("Job=%s not found. Assuming it was removed!!!\n"), jg->name); + } else { + job->globals = jg; /* Set globals pointer */ + } + } + SCHED_GLOBALS *schg; + foreach_dlist(schg, &sched_globals) { + SCHED *sched; + sched = GetSchedResWithName(schg->name); + if (!sched) { + Jmsg(NULL, M_INFO, 0, _("Schedule=%s not found. Assuming it was removed!!!\n"), schg->name); + } else { + sched->globals = schg; /* Set globals pointer */ + } + } } - /* Reset globals */ + /* Reset other globals */ set_working_directory(director->working_directory); FDConnectTimeout = director->FDConnectTimeout; SDConnectTimeout = director->SDConnectTimeout; @@ -582,13 +691,44 @@ void terminate_dird(int sig) print_memory_pool_stats(); } if (config) { - config->free_resources(); - free(config); + delete config; config = NULL; } term_ua_server(); term_msg(); /* terminate message handler */ cleanup_crypto(); + + P(daemon_msg_queue_mutex); + daemon_msg_queue->destroy(); + free(daemon_msg_queue); + V(daemon_msg_queue_mutex); + + if (reload_table) { + free(reload_table); + } + free(res_head); + res_head = NULL; + /* + * Now walk through resource globals tables and release them + */ + CLIENT_GLOBALS *cg; + foreach_dlist(cg, &client_globals) { + free(cg->name); + if (cg->SetIPaddress) { + free(cg->SetIPaddress); + } + free(cg); + } + STORE_GLOBALS *sg; + foreach_dlist(sg, &store_globals) { + free(sg->name); + free(sg); + } + JOB_GLOBALS *jg; + foreach_dlist(jg, &job_globals) { + free(jg->name); + free(jg); + } close_memory_pool(); /* release free memory in pool */ lmgr_cleanup_main(); sm_dump(false); @@ -690,15 +830,7 @@ static bool check_resources() int i; if (job->jobdefs) { - /* Handle Storage alists specifically */ JOB *jobdefs = job->jobdefs; - if (jobdefs->storage && !job->storage) { - STORE *st; - job->storage = New(alist(10, not_owned_by_alist)); - foreach_alist(st, jobdefs->storage) { - job->storage->append(st); - } - } /* Handle RunScripts alists specifically */ if (jobdefs->RunScripts) { RUNSCRIPT *rs, *elt; @@ -719,6 +851,7 @@ static bool check_resources() uint32_t *def_ivalue, *ivalue; /* integer value */ bool *def_bvalue, *bvalue; /* bool value */ int64_t *def_lvalue, *lvalue; /* 64 bit values */ + alist **def_avalue, **avalue; /* alist values */ uint32_t offset; Dmsg4(1400, "Job \"%s\", field \"%s\" bit=%d def=%d\n", @@ -762,9 +895,17 @@ static bool check_resources() * Handle alist resources */ } else if (job_items[i].handler == store_alist_res) { - if (bit_is_set(i, job->jobdefs->hdr.item_present)) { - set_bit(i, job->hdr.item_present); + void *elt; + + def_avalue = (alist **)((char *)(job->jobdefs) + offset); + avalue = (alist **)((char *)job + offset); + + *avalue = New(alist(10, not_owned_by_alist)); + + foreach_alist(elt, (*def_avalue)) { + (*avalue)->append(elt); } + set_bit(i, job->hdr.item_present); /* * Handle integer fields * Note, our store_bit does not handle bitmaped fields @@ -831,6 +972,16 @@ static bool check_resources() job->name()); OK = false; } + + /* Make sure the job doesn't use the Scratch Pool to start with */ + const char *name; + if (!check_pool(job->JobType, job->JobLevel, + job->pool, job->next_pool, &name)) { + Jmsg(NULL, M_FATAL, 0, + _("%s \"Scratch\" not valid in Job \"%s\".\n"), + name, job->name()); + OK = false; + } } /* End loop over Job res */ @@ -928,6 +1079,34 @@ static bool check_resources() } } + /* Loop over all pools, check PoolType */ + POOL *pool; + foreach_res(pool, R_POOL) { + if (!pool->pool_type) { + /* This case is checked by the parse engine, we should not */ + Jmsg(NULL, M_FATAL, 0, _("PoolType required in Pool resource \"%s\".\n"), pool->hdr.name); + OK = false; + continue; + } + if ((strcasecmp(pool->pool_type, NT_("backup")) != 0) && + (strcasecmp(pool->pool_type, NT_("copy")) != 0) && + (strcasecmp(pool->pool_type, NT_("cloned")) != 0) && + (strcasecmp(pool->pool_type, NT_("archive")) != 0) && + (strcasecmp(pool->pool_type, NT_("migration")) != 0) && + (strcasecmp(pool->pool_type, NT_("scratch")) != 0)) + { + Jmsg(NULL, M_FATAL, 0, _("Invalid PoolType \"%s\" in Pool resource \"%s\".\n"), pool->pool_type, pool->hdr.name); + OK = false; + } + + if (pool->NextPool && strcmp(pool->NextPool->name(), "Scratch") == 0) { + Jmsg(NULL, M_FATAL, 0, + _("NextPool \"Scratch\" not valid in Pool \"%s\".\n"), + pool->name()); + OK = false; + } + } + UnlockRes(); if (OK) { close_msg(NULL); /* close temp message handler */ @@ -942,11 +1121,14 @@ static bool check_resources() * - we can check the connection (mode=CHECK_CONNECTION) * - we can synchronize the catalog with the configuration (mode=UPDATE_CATALOG) * - we can synchronize, and fix old job records (mode=UPDATE_AND_FIX) + * - we hook up the Autochange children with the parent, and + * we hook the shared autochangers together. */ static bool check_catalog(cat_op mode) { bool OK = true; bool need_tls; + STORE *store, *ac_child; /* Loop over databases */ CAT *catalog; @@ -1034,7 +1216,6 @@ static bool check_catalog(cat_op mode) } /* Ensure basic storage record is in DB */ - STORE *store; foreach_res(store, R_STORAGE) { STORAGE_DBR sr; MEDIATYPE_DBR mtr; @@ -1100,6 +1281,38 @@ static bool check_catalog(cat_op mode) } } + /* Link up all the children for each changer */ + foreach_res(store, R_STORAGE) { + char sid[50]; + if (store->changer == store) { /* we are a real Autochanger */ + store->ac_group = get_pool_memory(PM_FNAME); + store->ac_group[0] = 0; + pm_strcat(store->ac_group, edit_int64(store->StorageId, sid)); + /* Now look for children who point to this storage */ + foreach_res(ac_child, R_STORAGE) { + if (ac_child != store && ac_child->changer == store) { + /* Found a child -- add StorageId */ + pm_strcat(store->ac_group, ","); + pm_strcat(store->ac_group, edit_int64(ac_child->StorageId, sid)); + } + } + } + } + + /* Link up all the shared storage devices */ + foreach_res(store, R_STORAGE) { + if (store->ac_group) { /* we are a real Autochanger */ + /* Now look for Shared Storage who point to this storage */ + foreach_res(ac_child, R_STORAGE) { + if (ac_child->shared_storage == store && ac_child->ac_group && + ac_child->shared_storage != ac_child) { + pm_strcat(store->ac_group, ","); + pm_strcat(store->ac_group, ac_child->ac_group); + } + } + } + } + /* Loop over all counters, defining them in each database */ /* Set default value in all counters */ COUNTER *counter; diff --git a/bacula/src/dird/dird.h b/bacula/src/dird/dird.h index 6f35c1373f..028ef04910 100644 --- a/bacula/src/dird/dird.h +++ b/bacula/src/dird/dird.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Includes specific to the Director * * Kern Sibbald, December MM - * */ #include "lib/ini.h" diff --git a/bacula/src/dird/dird_conf.c b/bacula/src/dird/dird_conf.c index 5cad2b9d6b..da85d7ae23 100644 --- a/bacula/src/dird/dird_conf.c +++ b/bacula/src/dird/dird_conf.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -36,7 +36,6 @@ * for the resource records. * * Kern Sibbald, January MM - * */ @@ -49,8 +48,14 @@ */ int32_t r_first = R_FIRST; int32_t r_last = R_LAST; -static RES *sres_head[R_LAST - R_FIRST + 1]; -RES **res_head = sres_head; +RES_HEAD **res_head; + +static pthread_mutex_t globals_mutex = PTHREAD_MUTEX_INITIALIZER; +dlist client_globals; +dlist job_globals; +dlist store_globals; +dlist sched_globals; + /* Imported subroutines */ extern void store_run(LEX *lc, RES_ITEM *item, int index, int pass); @@ -65,7 +70,8 @@ void store_level(LEX *lc, RES_ITEM *item, int index, int pass); void store_replace(LEX *lc, RES_ITEM *item, int index, int pass); void store_acl(LEX *lc, RES_ITEM *item, int index, int pass); void store_migtype(LEX *lc, RES_ITEM *item, int index, int pass); -static void store_device(LEX *lc, RES_ITEM *item, int index, int pass); +void store_ac_res(LEX *lc, RES_ITEM *item, int index, int pass); +void store_device(LEX *lc, RES_ITEM *item, int index, int pass); void store_actiononpurge(LEX *lc, RES_ITEM *item, int index, int pass); static void store_runscript_when(LEX *lc, RES_ITEM *item, int index, int pass); static void store_runscript_cmd(LEX *lc, RES_ITEM *item, int index, int pass); @@ -85,10 +91,240 @@ URES res_all; #endif int32_t res_all_size = sizeof(res_all); +/* Implementation of certain classes */ + +void CLIENT::create_client_globals() +{ + globals = (CLIENT_GLOBALS *)malloc(sizeof(CLIENT_GLOBALS)); + memset(globals, 0, sizeof(CLIENT_GLOBALS)); + globals->name = bstrdup(name()); + client_globals.append(globals); +} + +int32_t CLIENT::getNumConcurrentJobs() +{ + if (!globals) { + return 0; + } + return globals->NumConcurrentJobs; +} + +void CLIENT::setNumConcurrentJobs(int32_t num) +{ + P(globals_mutex); + if (!globals) { + create_client_globals(); + /* Copy .conf IP address and Enabled */ + globals->enabled = Enabled; + globals->SetIPaddress = bstrdup(client_address); + } + globals->NumConcurrentJobs = num; + V(globals_mutex); + ASSERT(num >= 0); + Dmsg2(200, "Set NumConcurrentJobs=%ld for Client %s\n", + num, globals->name); +} + +char *CLIENT::address() +{ + if (!globals) { + return client_address; + } + if (!globals->SetIPaddress) { + return client_address; + } + return globals->SetIPaddress; +} + +void CLIENT::setAddress(char *addr) +{ + P(globals_mutex); + if (!globals) { + create_client_globals(); + globals->enabled = Enabled; /* copy .conf variable */ + } + if (globals->SetIPaddress) { + free(globals->SetIPaddress); + } + globals->SetIPaddress = bstrdup(addr); + V(globals_mutex); +} + +bool CLIENT::is_enabled() +{ + if (!globals) { + return Enabled; + } + return globals->enabled; +} + +void CLIENT::setEnabled(bool val) +{ + P(globals_mutex); + if (!globals) { + create_client_globals(); + globals->SetIPaddress = bstrdup(client_address); /* copy .conf variable */ + } + globals->enabled = val; + V(globals_mutex); + Dmsg2(200, "Set Enabled=%d for Client %s\n", + val, globals->name); +} + +void JOB::create_job_globals() +{ + globals = (JOB_GLOBALS *)malloc(sizeof(JOB_GLOBALS)); + memset(globals, 0, sizeof(JOB_GLOBALS)); + globals->name = bstrdup(name()); + job_globals.append(globals); +} + +int32_t JOB::getNumConcurrentJobs() +{ + if (!globals) { + return 0; + } + return globals->NumConcurrentJobs; +} + +void JOB::setNumConcurrentJobs(int32_t num) +{ + P(globals_mutex); + if (!globals) { + create_job_globals(); + globals->enabled = Enabled; /* copy .conf variable */ + } + globals->NumConcurrentJobs = num; + V(globals_mutex); + ASSERT(num >= 0); + Dmsg2(200, "Set NumConcurrentJobs=%ld for Job %s\n", + num, globals->name); +} + +bool JOB::is_enabled() +{ + if (!globals) { + return Enabled; + } + return globals->enabled; +} + +void JOB::setEnabled(bool val) +{ + P(globals_mutex); + if (!globals) { + create_job_globals(); + } + globals->enabled = val; + V(globals_mutex); + Dmsg2(200, "Set Enabled=%d for Job %s\n", + val, globals->name); +} + +void STORE::create_store_globals() +{ + globals = (STORE_GLOBALS *)malloc(sizeof(STORE_GLOBALS)); + memset(globals, 0, sizeof(STORE_GLOBALS)); + globals->name = bstrdup(name()); + store_globals.append(globals); +} + +int32_t STORE::getNumConcurrentReadJobs() +{ + if (!globals) { + return 0; + } + return globals->NumConcurrentReadJobs; +} + +void STORE::setNumConcurrentReadJobs(int32_t num) +{ + P(globals_mutex); + if (!globals) { + create_store_globals(); + globals->enabled = Enabled; /* copy .conf variable */ + } + globals->NumConcurrentReadJobs = num; + V(globals_mutex); + Dmsg2(200, "Set NumConcurrentReadJobs=%ld for Store %s\n", + num, globals->name); + ASSERT(num >= 0); +} + +int32_t STORE::getNumConcurrentJobs() +{ + if (!globals) { + return 0; + } + return globals->NumConcurrentJobs; +} + +void STORE::setNumConcurrentJobs(int32_t num) +{ + P(globals_mutex); + if (!globals) { + create_store_globals(); + globals->enabled = Enabled; /* copy .conf variable */ + } + globals->NumConcurrentJobs = num; + V(globals_mutex); + Dmsg2(200, "Set numconcurrentJobs=%ld for Store %s\n", + num, globals->name); + ASSERT(num >= 0); +} + +bool STORE::is_enabled() +{ + if (!globals) { + return Enabled; + } + return globals->enabled; +} + +void STORE::setEnabled(bool val) +{ + P(globals_mutex); + if (!globals) { + create_store_globals(); + } + globals->enabled = val; + V(globals_mutex); + Dmsg2(200, "Set Enabled=%d for Storage %s\n", + val, globals->name); +} + +void SCHED::create_sched_globals() +{ + globals = (SCHED_GLOBALS *)malloc(sizeof(CLIENT_GLOBALS)); + memset(globals, 0, sizeof(SCHED_GLOBALS)); + globals->name = bstrdup(name()); + sched_globals.append(globals); +} + +bool SCHED::is_enabled() +{ + if (!globals) { + return Enabled; + } + return globals->enabled; +} + +void SCHED::setEnabled(bool val) +{ + P(globals_mutex); + if (!globals) { + create_sched_globals(); + } + globals->enabled = val; + V(globals_mutex); + Dmsg2(200, "Set Enabled=%d for Schedule %s\n", + val, globals->name); +} + /* * Definition of records permitted within each * resource with the routine to process the record - * information. + * information. NOTE! quoted names must be in lower case. */ /* * Director Resource @@ -109,7 +345,7 @@ static RES_ITEM dir_items[] = { {"ScriptsDirectory", store_dir, ITEM(res_dir.scripts_directory), 0, 0, 0}, {"PidDirectory", store_dir, ITEM(res_dir.pid_directory), 0, ITEM_REQUIRED, 0}, {"SubsysDirectory", store_dir, ITEM(res_dir.subsys_directory), 0, 0, 0}, - {"MaximumConcurrentJobs", store_pint32, ITEM(res_dir.MaxConcurrentJobs), 0, ITEM_DEFAULT, 1}, + {"MaximumConcurrentJobs", store_pint32, ITEM(res_dir.MaxConcurrentJobs), 0, ITEM_DEFAULT, 20}, {"MaximumReloadRequests", store_pint32, ITEM(res_dir.MaxReload), 0, ITEM_DEFAULT, 32}, {"MaximumConsoleConnections", store_pint32, ITEM(res_dir.MaxConsoleConnect), 0, ITEM_DEFAULT, 20}, {"Password", store_password, ITEM(res_dir.password), 0, ITEM_REQUIRED, 0}, @@ -128,6 +364,7 @@ static RES_ITEM dir_items[] = { {"TlsAllowedCn", store_alist_str, ITEM(res_dir.tls_allowed_cns), 0, 0, 0}, {"StatisticsRetention", store_time, ITEM(res_dir.stats_retention), 0, ITEM_DEFAULT, 60*60*24*31*12*5}, {"VerId", store_str, ITEM(res_dir.verid), 0, 0, 0}, + {"CommCompression", store_bool, ITEM(res_dir.comm_compression), 0, ITEM_DEFAULT, true}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -150,6 +387,8 @@ static RES_ITEM con_items[] = { {"FilesetAcl", store_acl, ITEM(res_con.ACL_lists), FileSet_ACL, 0, 0}, {"CatalogAcl", store_acl, ITEM(res_con.ACL_lists), Catalog_ACL, 0, 0}, {"WhereAcl", store_acl, ITEM(res_con.ACL_lists), Where_ACL, 0, 0}, + {"RestoreClientAcl", store_acl, ITEM(res_con.ACL_lists), RestoreClient_ACL, 0, 0}, + {"BackupClientAcl", store_acl, ITEM(res_con.ACL_lists), BackupClient_ACL, 0, 0}, {"PluginOptionsAcl", store_acl, ITEM(res_con.ACL_lists), PluginOptions_ACL, 0, 0}, {"TlsAuthenticate", store_bool, ITEM(res_con.tls_authenticate), 0, 0, 0}, {"TlsEnable", store_bool, ITEM(res_con.tls_enable), 0, 0, 0}, @@ -174,8 +413,8 @@ static RES_ITEM con_items[] = { static RES_ITEM cli_items[] = { {"Name", store_name, ITEM(res_client.hdr.name), 0, ITEM_REQUIRED, 0}, {"Description", store_str, ITEM(res_client.hdr.desc), 0, 0, 0}, - {"fdaddress", store_str, ITEM(res_client.address), 0, 0, 0}, - {"Address", store_str, ITEM(res_client.address), 0, ITEM_REQUIRED, 0}, + {"fdaddress", store_str, ITEM(res_client.client_address), 0, 0, 0}, + {"Address", store_str, ITEM(res_client.client_address), 0, ITEM_REQUIRED, 0}, {"FdPort", store_pint32, ITEM(res_client.FDport), 0, ITEM_DEFAULT, 9102}, {"fdpassword", store_password, ITEM(res_client.password), 0, 0, 0}, {"Password", store_password, ITEM(res_client.password), 0, ITEM_REQUIRED, 0}, @@ -197,7 +436,7 @@ static RES_ITEM cli_items[] = { {"TlsKey", store_dir, ITEM(res_client.tls_keyfile), 0, 0, 0}, {"TlsAllowedCn", store_alist_str, ITEM(res_client.tls_allowed_cns), 0, 0, 0}, {"MaximumBandwidthPerJob", store_speed, ITEM(res_client.max_bandwidth), 0, 0, 0}, - {"Enabled", store_bool, ITEM(res_client.enabled), 0, ITEM_DEFAULT, true}, + {"Enabled", store_bool, ITEM(res_client.Enabled), 0, ITEM_DEFAULT, true}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -216,8 +455,14 @@ static RES_ITEM store_items[] = { {"Password", store_password, ITEM(res_store.password), 0, ITEM_REQUIRED, 0}, {"Device", store_device, ITEM(res_store.device), R_DEVICE, ITEM_REQUIRED, 0}, {"MediaType", store_strname, ITEM(res_store.media_type), 0, ITEM_REQUIRED, 0}, + /* _bool, + * Big kludge, these two autochanger definitions must be in + * this order and together. + */ + {"Autochanger", store_ac_res, ITEM(res_store.changer), 0, ITEM_DEFAULT, 0}, {"Autochanger", store_bool, ITEM(res_store.autochanger), 0, ITEM_DEFAULT, false}, - {"Enabled", store_bool, ITEM(res_store.enabled), 0, ITEM_DEFAULT, true}, + {"SharedStorage", store_ac_res, ITEM(res_store.shared_storage), 1, ITEM_DEFAULT, 0}, + {"Enabled", store_bool, ITEM(res_store.Enabled), 0, ITEM_DEFAULT, true}, {"AllowCompression", store_bool, ITEM(res_store.AllowCompress), 0, ITEM_DEFAULT, true}, {"HeartbeatInterval", store_time, ITEM(res_store.heartbeat_interval), 0, ITEM_DEFAULT, 5 * 60}, {"MaximumConcurrentJobs", store_pint32, ITEM(res_store.MaxConcurrentJobs), 0, ITEM_DEFAULT, 1}, @@ -251,12 +496,11 @@ static RES_ITEM cat_items[] = { {"User", store_str, ITEM(res_cat.db_user), 0, 0, 0}, {"DbName", store_str, ITEM(res_cat.db_name), 0, ITEM_REQUIRED, 0}, {"dbdriver", store_str, ITEM(res_cat.db_driver), 0, 0, 0}, - {"DbSocket", store_str, ITEM(res_cat.db_socket), 0, 0, 0}, {"dbsslkey", store_str, ITEM(res_cat.db_ssl_key), 0, 0, 0}, {"dbsslcert", store_str, ITEM(res_cat.db_ssl_cert), 0, 0, 0}, {"dbsslca", store_str, ITEM(res_cat.db_ssl_ca), 0, 0, 0}, {"dbsslcapath", store_str, ITEM(res_cat.db_ssl_capath), 0, 0, 0}, - {"dbsslcipher", store_str, ITEM(res_cat.db_ssl_cipher), 0, 0, 0}, + {"DbSocket", store_str, ITEM(res_cat.db_socket), 0, 0, 0}, /* Turned off for the moment */ {"MultipleConnections", store_bit, ITEM(res_cat.mult_db_connections), 0, 0, 0}, {"DisableBatchInsert", store_bool, ITEM(res_cat.disable_batch_insert), 0, ITEM_DEFAULT, false}, @@ -318,20 +562,24 @@ RES_ITEM job_items[] = { {"PruneFiles", store_bool, ITEM(res_job.PruneFiles), 0, ITEM_DEFAULT, false}, {"PruneVolumes",store_bool, ITEM(res_job.PruneVolumes), 0, ITEM_DEFAULT, false}, {"PurgeMigrationJob", store_bool, ITEM(res_job.PurgeMigrateJob), 0, ITEM_DEFAULT, false}, - {"Enabled", store_bool, ITEM(res_job.enabled), 0, ITEM_DEFAULT, true}, + {"Enabled", store_bool, ITEM(res_job.Enabled), 0, ITEM_DEFAULT, true}, {"SnapshotRetention", store_time, ITEM(res_job.SnapRetention), 0, ITEM_DEFAULT, 0}, {"SpoolAttributes",store_bool, ITEM(res_job.SpoolAttributes), 0, ITEM_DEFAULT, true}, {"SpoolData", store_bool, ITEM(res_job.spool_data), 0, ITEM_DEFAULT, false}, {"SpoolSize", store_size64, ITEM(res_job.spool_size), 0, 0, 0}, {"ReRunFailedLevels", store_bool, ITEM(res_job.rerun_failed_levels), 0, ITEM_DEFAULT, false}, {"PreferMountedVolumes", store_bool, ITEM(res_job.PreferMountedVolumes), 0, ITEM_DEFAULT, true}, - {"RunBeforeJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, - {"RunAfterJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, - {"RunAfterFailedJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, - {"ClientRunBeforeJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, - {"ClientRunAfterJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, - {"ConsoleRunBeforeJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, - {"ConsoleRunAfterJob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + /* + * JSON tools skip Directive in lowercase. They are deprecated or + * are synonym with an other one that follows. Like User and dbuser. + */ + {"runbeforejob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + {"runafterjob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + {"runafterfailedjob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + {"clientrunbeforejob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + {"clientrunafterjob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + {"consolerunbeforejob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, + {"consolerunafterjob", store_short_runscript, ITEM(res_job.RunScripts), 0, 0, 0}, {"Runscript", store_runscript, ITEM(res_job.RunScripts), 0, ITEM_NO_EQUALS, 0}, {"MaximumConcurrentJobs", store_pint32, ITEM(res_job.MaxConcurrentJobs), 0, ITEM_DEFAULT, 1}, {"MaximumSpawnedJobs", store_pint32, ITEM(res_job.MaxSpawnedJobs), 0, ITEM_DEFAULT, 600}, @@ -340,6 +588,7 @@ RES_ITEM job_items[] = { {"RescheduleInterval", store_time, ITEM(res_job.RescheduleInterval), 0, ITEM_DEFAULT, 60 * 30}, {"RescheduleTimes", store_pint32, ITEM(res_job.RescheduleTimes), 0, 0, 0}, {"Priority", store_pint32, ITEM(res_job.Priority), 0, ITEM_DEFAULT, 10}, + {"BackupsToKeep", store_pint32, ITEM(res_job.BackupsToKeep), 0, ITEM_DEFAULT, 0}, {"AllowMixedPriority", store_bool, ITEM(res_job.allow_mixed_priority), 0, ITEM_DEFAULT, false}, {"WritePartAfterJob", store_bool, ITEM(res_job.write_part_after_job), 0, ITEM_DEFAULT, true}, {"SelectionPattern", store_str, ITEM(res_job.selection_pattern), 0, 0, 0}, @@ -350,6 +599,7 @@ RES_ITEM job_items[] = { {"CancelLowerLevelDuplicates", store_bool, ITEM(res_job.CancelLowerLevelDuplicates), 0, ITEM_DEFAULT, false}, {"CancelQueuedDuplicates", store_bool, ITEM(res_job.CancelQueuedDuplicates), 0, ITEM_DEFAULT, false}, {"CancelRunningDuplicates", store_bool, ITEM(res_job.CancelRunningDuplicates), 0, ITEM_DEFAULT, false}, + {"DeleteConsolidatedJobs", store_bool, ITEM(res_job.DeleteConsolidatedJobs), 0, ITEM_DEFAULT, false}, {"PluginOptions", store_str, ITEM(res_job.PluginOptions), 0, 0, 0}, {"Base", store_alist_res, ITEM(res_job.base), R_JOB, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} @@ -379,7 +629,7 @@ static RES_ITEM sch_items[] = { {"Name", store_name, ITEM(res_sch.hdr.name), 0, ITEM_REQUIRED, 0}, {"Description", store_str, ITEM(res_sch.hdr.desc), 0, 0, 0}, {"Run", store_run, ITEM(res_sch.run), 0, 0, 0}, - {"Enabled", store_bool, ITEM(res_sch.enabled), 0, ITEM_DEFAULT, true}, + {"Enabled", store_bool, ITEM(res_sch.Enabled), 0, ITEM_DEFAULT, true}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -405,6 +655,7 @@ static RES_ITEM pool_items[] = { {"MaximumVolumeFiles", store_pint32, ITEM(res_pool.MaxVolFiles), 0, 0, 0}, {"MaximumVolumeBytes", store_size64, ITEM(res_pool.MaxVolBytes), 0, 0, 0}, {"CatalogFiles", store_bool, ITEM(res_pool.catalog_files), 0, ITEM_DEFAULT, true}, + {"CacheRetention", store_time, ITEM(res_pool.CacheRetention), 0, 0, 0}, {"VolumeRetention", store_time, ITEM(res_pool.VolRetention), 0, ITEM_DEFAULT, 60*60*24*365}, {"VolumeUseDuration", store_time, ITEM(res_pool.VolUseDuration), 0, 0, 0}, {"MigrationTime", store_time, ITEM(res_pool.MigrationTime), 0, 0, 0}, @@ -465,6 +716,7 @@ RES_TABLE resources[] = { {"Console", con_items, R_CONSOLE}, {"JobDefs", job_items, R_JOBDEFS}, {"Device", NULL, R_DEVICE}, /* info obtained from SD */ + {"Autochanger", store_items, R_AUTOCHANGER}, /* alias for R_STORAGE */ {NULL, NULL, 0} }; @@ -532,7 +784,7 @@ s_jt migtypes[] = { /* Options permitted in Restore replace= */ -struct s_kw ReplaceOptions[] = { +s_kw ReplaceOptions[] = { {"Always", REPLACE_ALWAYS}, {"IfNewer", REPLACE_IFNEWER}, {"IfOlder", REPLACE_IFOLDER}, @@ -569,6 +821,7 @@ const char *level_to_str(int level) /* Dump contents of resource */ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, ...), void *sock) { + RES *next; URES *res = (URES *)ares; bool recurse = true; char ed1[100], ed2[100], ed3[100]; @@ -619,13 +872,13 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, break; case R_CLIENT: - if (!acl_access_ok(ua, Client_ACL, res->res_client.hdr.name)) { + if (!acl_access_ok(ua, Client_ACL, res->res_client.name())) { break; } - sendit(sock, _("Client: Name=%s Enabled=%d Address=%s FDport=%d MaxJobs=%u\n"), - res->res_client.hdr.name, res->res_client.enabled, - res->res_client.address, res->res_client.FDport, - res->res_client.MaxConcurrentJobs); + sendit(sock, _("Client: Name=%s Enabled=%d Address=%s FDport=%d MaxJobs=%u NumJobs=%u\n"), + res->res_client.name(), res->res_client.is_enabled(), + res->res_client.address(), res->res_client.FDport, + res->res_client.MaxConcurrentJobs, res->res_client.getNumConcurrentJobs()); sendit(sock, _(" JobRetention=%s FileRetention=%s AutoPrune=%d\n"), edit_utime(res->res_client.JobRetention, ed1, sizeof(ed1)), edit_utime(res->res_client.FileRetention, ed2, sizeof(ed2)), @@ -656,14 +909,17 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, dev->VolumeName, dev->MediaType); break; + case R_AUTOCHANGER: case R_STORAGE: if (!acl_access_ok(ua, Storage_ACL, res->res_store.hdr.name)) { break; } - sendit(sock, _("Storage: name=%s address=%s SDport=%d MaxJobs=%u\n" + sendit(sock, _("%s: name=%s address=%s SDport=%d MaxJobs=%u NumJobs=%u\n" " DeviceName=%s MediaType=%s StorageId=%s Autochanger=%d\n"), + res->res_store.changer == &res->res_store ? "Autochanger" : "Storage", res->res_store.hdr.name, res->res_store.address, res->res_store.SDport, res->res_store.MaxConcurrentJobs, + res->res_store.getNumConcurrentJobs(), res->res_store.dev_name(), res->res_store.media_type, edit_int64(res->res_store.StorageId, ed1), @@ -671,6 +927,15 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, if (res->res_store.fd_storage_address) { sendit(sock, " FDStorageAddress=%s\n", res->res_store.fd_storage_address); } + if (res->res_store.ac_group) { + STORE *shstore = res->res_store.shared_storage; + sendit(sock, " AC group=%s ShareStore=%s\n", res->res_store.ac_group, + shstore?shstore->name():"*none*"); + } + if (res->res_store.changer && res->res_store.changer != &res->res_store) { + sendit(sock, _(" Parent --> ")); + dump_resource(-R_STORAGE, (RES *)res->res_store.changer, sendit, sock); + } break; case R_CATALOG: @@ -694,9 +959,10 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, type == R_JOB ? _("Job") : _("JobDefs"), res->res_job.hdr.name, res->res_job.JobType, level_to_str(res->res_job.JobLevel), res->res_job.Priority, - res->res_job.enabled); - sendit(sock, _(" MaxJobs=%u Resched=%d Times=%d Interval=%s Spool=%d WritePartAfterJob=%d\n"), + res->res_job.is_enabled()); + sendit(sock, _(" MaxJobs=%u NumJobs=%u Resched=%d Times=%d Interval=%s Spool=%d WritePartAfterJob=%d\n"), res->res_job.MaxConcurrentJobs, + res->res_job.getNumConcurrentJobs(), res->res_job.RescheduleOnError, res->res_job.RescheduleTimes, edit_uint64_with_commas(res->res_job.RescheduleInterval, ed1), res->res_job.spool_data, res->res_job.write_part_after_job); @@ -796,6 +1062,10 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, sendit(sock, _(" --> DifferentialBackup")); dump_resource(-R_POOL, (RES *)res->res_job.diff_pool, sendit, sock); } + if (res->res_job.next_pool) { + sendit(sock, _(" --> Next")); /* Pool will be added by dump_resource */ + dump_resource(-R_POOL, (RES *)res->res_job.next_pool, sendit, sock); + } if (res->res_job.verify_job) { sendit(sock, _(" --> ")); dump_resource(-type, (RES *)res->res_job.verify_job, sendit, sock); @@ -821,7 +1091,7 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, if (!acl_access_ok(ua, FileSet_ACL, res->res_fs.hdr.name)) { break; } - sendit(sock, _("FileSet: name=%s\n"), res->res_fs.hdr.name); + sendit(sock, _("FileSet: name=%s IgnoreFileSetChanges=%d\n"), res->res_fs.hdr.name, res->res_fs.ignore_fs_changes); for (i=0; ires_fs.num_includes; i++) { INCEXE *incexe = res->res_fs.include_items[i]; for (j=0; jnum_opts; j++) { @@ -916,7 +1186,7 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, RUN *run = res->res_sch.run; char buf[1000], num[30]; sendit(sock, _("Schedule: Name=%s Enabled=%d\n"), - res->res_sch.hdr.name, res->res_sch.enabled); + res->res_sch.hdr.name, res->res_sch.is_enabled()); if (!run) { break; } @@ -987,6 +1257,10 @@ next_run: sendit(sock, _(" --> ")); dump_resource(-R_POOL, (RES *)run->pool, sendit, sock); } + if (run->next_pool) { + sendit(sock, _(" --> Next")); /* Pool will be added by dump_resource */ + dump_resource(-R_POOL, (RES *)run->next_pool, sendit, sock); + } if (run->storage) { sendit(sock, _(" --> ")); dump_resource(-R_STORAGE, (RES *)run->storage, sendit, sock); @@ -1035,6 +1309,8 @@ next_run: edit_utime(res->res_pool.MigrationTime, ed1, sizeof(ed1)), edit_uint64(res->res_pool.MigrationHighBytes, ed2), edit_uint64(res->res_pool.MigrationLowBytes, ed3)); + sendit(sock, _(" CacheRetention=%s\n"), + edit_utime(res->res_pool.CacheRetention, ed1, sizeof(ed1))); sendit(sock, _(" JobRetention=%s FileRetention=%s\n"), edit_utime(res->res_pool.JobRetention, ed1, sizeof(ed1)), edit_utime(res->res_pool.FileRetention, ed2, sizeof(ed2))); @@ -1079,10 +1355,12 @@ next_run: sendit(sock, _("Unknown resource type %d in dump_resource.\n"), type); break; } - if (recurse && res->res_dir.hdr.next) { - dump_resource(type, res->res_dir.hdr.next, sendit, sock); + if (recurse) { + next = GetNextRes(0, (RES *)res); + if (next) { + dump_resource(type, next, sendit, sock); + } } - } /* @@ -1135,7 +1413,6 @@ static void free_incexe(INCEXE *incexe) void free_resource(RES *rres, int type) { int num; - RES *nres; URES *res = (URES *)rres; if (res == NULL) { @@ -1144,7 +1421,6 @@ void free_resource(RES *rres, int type) Dmsg3(200, "type=%d res=%p name=%s\n", type, res, res->res_dir.hdr.name); /* common stuff -- free the resource name and description */ - nres = (RES *)res->res_dir.hdr.next; if (res->res_dir.hdr.name) { free(res->res_dir.hdr.name); } @@ -1242,8 +1518,8 @@ void free_resource(RES *rres, int type) } break; case R_CLIENT: - if (res->res_client.address) { - free(res->res_client.address); + if (res->res_client.client_address) { + free(res->res_client.client_address); } if (res->res_client.fd_storage_address) { free(res->res_client.fd_storage_address); @@ -1270,6 +1546,7 @@ void free_resource(RES *rres, int type) delete res->res_client.tls_allowed_cns; } break; + case R_AUTOCHANGER: case R_STORAGE: if (res->res_store.address) { free(res->res_store.address); @@ -1283,6 +1560,9 @@ void free_resource(RES *rres, int type) if (res->res_store.media_type) { free(res->res_store.media_type); } + if (res->res_store.ac_group) { + free_pool_memory(res->res_store.ac_group); + } if (res->res_store.device) { delete res->res_store.device; } @@ -1438,9 +1718,6 @@ void free_resource(RES *rres, int type) if (res) { free(res); } - if (nres) { - free_resource(nres, type); - } } /* @@ -1449,7 +1726,7 @@ void free_resource(RES *rres, int type) * pointers because they may not have been defined until * later in pass 1. */ -void save_resource(int type, RES_ITEM *items, int pass) +bool save_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { URES *res; int rindex = type - r_first; @@ -1464,13 +1741,15 @@ void save_resource(int type, RES_ITEM *items, int pass) for (i=0; items[i].name; i++) { if (items[i].flags & ITEM_REQUIRED) { if (!bit_is_set(i, res_all.res_dir.hdr.item_present)) { - Emsg2(M_ERROR_TERM, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), items[i].name, resources[rindex].name); + return false; } } /* If this triggers, take a look at lib/parse_conf.h */ if (i >= MAX_RES_ITEMS) { - Emsg1(M_ERROR_TERM, 0, _("Too many directive in \"%s\" resource\n"), resources[rindex].name); + Mmsg(config->m_errmsg, _("Too many directives in \"%s\" resource\n"), resources[rindex].name); + return false; } } } else if (type == R_JOB) { @@ -1479,8 +1758,9 @@ void save_resource(int type, RES_ITEM *items, int pass) */ if (items[0].flags & ITEM_REQUIRED) { if (!bit_is_set(0, res_all.res_dir.hdr.item_present)) { - Emsg2(M_ERROR_TERM, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), - items[0].name, resources[rindex].name); + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + items[0].name, resources[rindex].name); + return false; } } } @@ -1510,7 +1790,8 @@ void save_resource(int type, RES_ITEM *items, int pass) case R_POOL: /* Find resource saved in pass 1 */ if ((res = (URES *)GetResWithName(R_POOL, res_all.res_con.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Pool resource %s\n"), res_all.res_con.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Pool resource %s\n"), res_all.res_con.hdr.name); + return false; } /* Explicitly copy resource pointers from this pass (res_all) */ res->res_pool.NextPool = res_all.res_pool.NextPool; @@ -1521,30 +1802,43 @@ void save_resource(int type, RES_ITEM *items, int pass) break; case R_CONSOLE: if ((res = (URES *)GetResWithName(R_CONSOLE, res_all.res_con.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Console resource %s\n"), res_all.res_con.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Console resource %s\n"), res_all.res_con.hdr.name); + return false; } res->res_con.tls_allowed_cns = res_all.res_con.tls_allowed_cns; break; case R_DIRECTOR: if ((res = (URES *)GetResWithName(R_DIRECTOR, res_all.res_dir.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Director resource %s\n"), res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Director resource %s\n"), res_all.res_dir.hdr.name); + return false; } res->res_dir.messages = res_all.res_dir.messages; res->res_dir.tls_allowed_cns = res_all.res_dir.tls_allowed_cns; break; + case R_AUTOCHANGER: /* alias for R_STORAGE */ case R_STORAGE: + type = R_STORAGE; /* force Storage type */ if ((res = (URES *)GetResWithName(type, res_all.res_store.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Storage resource %s\n"), - res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Storage resource %s\n"), + res_all.res_dir.hdr.name); + return false; } /* we must explicitly copy the device alist pointer */ res->res_store.device = res_all.res_store.device; + res->res_store.changer = res_all.res_store.changer; + res->res_store.shared_storage = res_all.res_store.shared_storage; + res->res_store.autochanger = res_all.res_store.autochanger; + if (strcasecmp(resources[rindex].name, "autochanger") == 0) { + res->res_store.changer = &res->res_store; + res->res_store.autochanger = true; + } break; case R_JOB: case R_JOBDEFS: if ((res = (URES *)GetResWithName(type, res_all.res_dir.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Job resource %s\n"), - res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Job resource %s\n"), + res_all.res_dir.hdr.name); + return false; } res->res_job.messages = res_all.res_job.messages; res->res_job.schedule = res_all.res_job.schedule; @@ -1594,15 +1888,17 @@ void save_resource(int type, RES_ITEM *items, int pass) break; case R_COUNTER: if ((res = (URES *)GetResWithName(R_COUNTER, res_all.res_counter.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Counter resource %s\n"), res_all.res_counter.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Counter resource %s\n"), res_all.res_counter.hdr.name); + return false; } res->res_counter.Catalog = res_all.res_counter.Catalog; res->res_counter.WrapCounter = res_all.res_counter.WrapCounter; break; case R_CLIENT: - if ((res = (URES *)GetResWithName(R_CLIENT, res_all.res_client.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Client resource %s\n"), res_all.res_client.hdr.name); + if ((res = (URES *)GetResWithName(R_CLIENT, res_all.res_client.name())) == NULL) { + Mmsg(config->m_errmsg, _("Cannot find Client resource %s\n"), res_all.res_client.name()); + return false; } res->res_client.catalog = res_all.res_client.catalog; res->res_client.tls_allowed_cns = res_all.res_client.tls_allowed_cns; @@ -1614,8 +1910,9 @@ void save_resource(int type, RES_ITEM *items, int pass) * in by run_conf.c during pass 2, so here we jam the pointer * into the Schedule resource. */ - if ((res = (URES *)GetResWithName(R_SCHEDULE, res_all.res_client.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Schedule resource %s\n"), res_all.res_client.hdr.name); + if ((res = (URES *)GetResWithName(R_SCHEDULE, res_all.res_client.name())) == NULL) { + Mmsg(config->m_errmsg, _("Cannot find Schedule resource %s\n"), res_all.res_client.name()); + return false; } res->res_sch.run = res_all.res_sch.run; break; @@ -1635,7 +1932,13 @@ void save_resource(int type, RES_ITEM *items, int pass) free(res_all.res_dir.hdr.desc); res_all.res_dir.hdr.desc = NULL; } - return; + return true; + } + + /* R_AUTOCHANGER is alias so turn it into an R_STORAGE */ + if (type == R_AUTOCHANGER) { + type = R_STORAGE; + rindex = type - r_first; } /* @@ -1686,32 +1989,11 @@ void save_resource(int type, RES_ITEM *items, int pass) } /* Common */ if (!error) { - res = (URES *)malloc(size); - memcpy(res, &res_all, size); - if (!res_head[rindex]) { - res_head[rindex] = (RES *)res; /* store first entry */ - Dmsg3(900, "Inserting first %s res: %s index=%d\n", res_to_str(type), - res->res_dir.hdr.name, rindex); - } else { - RES *next, *last; - if (res->res_dir.hdr.name == NULL) { - Emsg1(M_ERROR_TERM, 0, _("A Name directive is required in \"%s\" resource, but not found.\n"), - resources[rindex].name); - } - /* Add new res to end of chain */ - for (last=next=res_head[rindex]; next; next=next->next) { - last = next; - if (strcmp(next->name, res->res_dir.hdr.name) == 0) { - Emsg2(M_ERROR_TERM, 0, - _("Attempt to define second \"%s\" resource named \"%s\" is not permitted.\n"), - resources[rindex].name, res->res_dir.hdr.name); - } - } - last->next = (RES *)res; - Dmsg4(900, _("Inserting %s res: %s index=%d pass=%d\n"), res_to_str(type), - res->res_dir.hdr.name, rindex, pass); + if (!config->insert_res(rindex, size)) { + return false; } } + return true; } void store_actiononpurge(LEX *lc, RES_ITEM *item, int index, int pass) @@ -1728,46 +2010,93 @@ void store_actiononpurge(LEX *lc, RES_ITEM *item, int index, int pass) set_bit(index, res_all.hdr.item_present); } +/* + * Store an autochanger resource. Used by Autochanger and + * SharedStorage direcives. + */ +void store_ac_res(LEX *lc, RES_ITEM *item, int index, int pass) +{ + RES *res; + RES_ITEM *next = item + 1; + + lex_get_token(lc, T_NAME); + Dmsg1(100, "Got name=%s\n", lc->str); + /* + * For backward compatibility, if yes/no, set the next item + */ + if (strcasecmp(item->name, "autochanger") == 0) { + if (strcasecmp(lc->str, "yes") == 0 || strcasecmp(lc->str, "true") == 0) { + *(bool *)(next->value) = true; + Dmsg2(100, "Item=%s got value=%s\n", item->name, lc->str); + scan_to_eol(lc); + return; + } else if (strcasecmp(lc->str, "no") == 0 || strcasecmp(lc->str, "false") == 0) { + *(bool *)(next->value) = false; + Dmsg2(100, "Item=%s got value=%s\n", item->name, lc->str); + scan_to_eol(lc); + return; + } + } + Dmsg2(100, "Item=%s got value=%s\n", item->name, lc->str); + + if (pass == 2) { + res = GetResWithName(R_STORAGE, lc->str); + if (res == NULL) { + scan_err3(lc, _("Could not find Storage Resource %s referenced on line %d : %s\n"), + lc->str, lc->line_no, lc->line); + return; + } + if (*(item->value)) { + scan_err3(lc, _("Attempt to redefine Storage resource \"%s\" referenced on line %d : %s\n"), + item->name, lc->line_no, lc->line); + return; + } + Dmsg2(100, "Store %s value=%p\n", lc->str, res); + *(item->value) = (char *)res; + *(bool *)(next->value) = true; + } + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); +} + + /* * Store Device. Note, the resource is created upon the * first reference. The details of the resource are obtained * later from the SD. */ -static void store_device(LEX *lc, RES_ITEM *item, int index, int pass) +void store_device(LEX *lc, RES_ITEM *item, int index, int pass) { - URES *res; int rindex = R_DEVICE - r_first; int size = sizeof(DEVICE); - bool found = false; if (pass == 1) { + URES *ures; + RES *res; + lex_get_token(lc, T_NAME); - if (!res_head[rindex]) { - res = (URES *)malloc(size); - memset(res, 0, size); - res->res_dev.hdr.name = bstrdup(lc->str); - res_head[rindex] = (RES *)res; /* store first entry */ - Dmsg3(900, "Inserting first %s res: %s index=%d\n", res_to_str(R_DEVICE), - res->res_dir.hdr.name, rindex); + rblist *list = res_head[rindex]->res_list; + ures = (URES *)malloc(size); + memset(ures, 0, size); + ures->res_dev.hdr.name = bstrdup(lc->str); + res = (RES *)ures; + if (list->empty()) { + list->insert(res, res_compare); + res_head[rindex]->first = res; + res_head[rindex]->last = res; } else { - RES *next; - /* See if it is already defined */ - for (next=res_head[rindex]; next->next; next=next->next) { - if (strcmp(next->name, lc->str) == 0) { - found = true; - break; - } - } - if (!found) { - res = (URES *)malloc(size); - memset(res, 0, size); - res->res_dev.hdr.name = bstrdup(lc->str); - next->next = (RES *)res; - Dmsg4(900, "Inserting %s res: %s index=%d pass=%d\n", res_to_str(R_DEVICE), - res->res_dir.hdr.name, rindex, pass); + RES *item, *prev; + prev = res_head[rindex]->last; + item = (RES *)list->insert(res, res_compare); + if (item == res) { + prev->res_next = res; + res_head[rindex]->last = res; + } else { + /* res not inserted */ + free(ures->res_dev.hdr.name); + free(ures); } } - scan_to_eol(lc); set_bit(index, res_all.hdr.item_present); } else { @@ -2159,7 +2488,7 @@ void store_runscript(LEX *lc, RES_ITEM *item, int index, int pass) /* callback function for edit_job_codes */ /* See ../lib/util.c, function edit_job_codes, for more remaining codes */ -extern "C" char *job_code_callback_director(JCR *jcr, const char* param) +extern "C" char *job_code_callback_director(JCR *jcr, const char* param, char *buf, int buflen) { static char yes[] = "yes"; static char no[] = "no"; @@ -2175,8 +2504,8 @@ extern "C" char *job_code_callback_director(JCR *jcr, const char* param) } break; case 'h': - if (jcr->client && jcr->client->address) { - return jcr->client->address; + if (jcr->client && jcr->client->address()) { + return jcr->client->address(); } break; case 'p': @@ -2195,6 +2524,16 @@ extern "C" char *job_code_callback_director(JCR *jcr, const char* param) return my_name; case 'C': return jcr->cloned ? yes : no; + case 'I': + if (buflen >= 50) { + if (jcr->wjcr) { + edit_uint64(jcr->wjcr->JobId, buf); + return buf; + } else { + edit_uint64(0, buf); + return buf; + } + } } return nothing; } @@ -2202,6 +2541,6 @@ extern "C" char *job_code_callback_director(JCR *jcr, const char* param) bool parse_dir_config(CONFIG *config, const char *configfile, int exit_code) { config->init(configfile, NULL, exit_code, (void *)&res_all, res_all_size, - r_first, r_last, resources, res_head); + r_first, r_last, resources, &res_head); return config->parse_config(); } diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index aeced6c9b3..af07887866 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Director specific configuration and defines * * Kern Sibbald, Feb MM - * */ /* NOTE: #includes at the end of this file */ @@ -44,6 +42,7 @@ enum { R_JOBDEFS, R_DEVICE, /* This is the real last device class */ + R_AUTOCHANGER, /* Alias for R_STORAGE after R_LAST */ R_FIRST = R_DIRECTOR, R_LAST = R_DEVICE /* keep this updated */ }; @@ -122,6 +121,7 @@ public: alist *tls_allowed_cns; /* TLS Allowed Clients */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ utime_t stats_retention; /* Stats retention period in seconds */ + bool comm_compression; /* Enable comm line compression */ bool tls_authenticate; /* Authenticated with TLS */ bool tls_enable; /* Enable TLS */ bool tls_require; /* Require TLS */ @@ -183,7 +183,10 @@ enum { Catalog_ACL, Where_ACL, PluginOptions_ACL, - Num_ACL /* keep last */ + RestoreClient_ACL, + BackupClient_ACL, + Directory_ACL, /* List of directories that can be accessed in the restore tree */ + Num_ACL /* keep last */ }; /* @@ -243,6 +246,14 @@ public: inline char *CAT::name() const { return hdr.name; } +class CLIENT_GLOBALS { +public: + dlink link; /* double link */ + const char *name; /* resource name */ + int32_t NumConcurrentJobs; /* number of concurrent jobs running */ + char *SetIPaddress; /* address from SetIP command */ + bool enabled; /* Enabled/disabled */ +}; /* * Client Resource @@ -250,19 +261,18 @@ inline char *CAT::name() const { return hdr.name; } */ class CLIENT { public: - RES hdr; - + RES hdr; + CLIENT_GLOBALS *globals; /* global variables */ uint32_t FDport; /* Where File daemon listens */ utime_t FileRetention; /* file retention period in seconds */ utime_t JobRetention; /* job retention period in seconds */ utime_t SnapRetention; /* Snapshot retention period in seconds */ utime_t heartbeat_interval; /* Interval to send heartbeats */ - char *address; + char *client_address; /* Client address from .conf file */ char *fd_storage_address; /* Storage address to use from FD side */ char *password; CAT *catalog; /* Catalog resource */ int32_t MaxConcurrentJobs; /* Maximum concurrent jobs */ - int32_t NumConcurrentJobs; /* number of concurrent jobs running */ char *tls_ca_certfile; /* TLS CA Certificate File */ char *tls_ca_certdir; /* TLS CA Certificate Directory */ char *tls_certfile; /* TLS Client Certificate File */ @@ -272,18 +282,35 @@ public: bool tls_authenticate; /* Authenticated with TLS */ bool tls_enable; /* Enable TLS */ bool tls_require; /* Require TLS */ - bool enabled; /* Set if client enabled */ + bool Enabled; /* Set if client enabled */ bool AutoPrune; /* Do automatic pruning? */ bool sd_calls_client; /* SD calls the client */ int64_t max_bandwidth; /* Limit speed on this client */ /* Methods */ char *name() const; + void create_client_globals(); + int32_t getNumConcurrentJobs(); + void setNumConcurrentJobs(int32_t num); + char *address(); + void setAddress(char *addr); + bool is_enabled(); + void setEnabled(bool val); }; inline char *CLIENT::name() const { return hdr.name; } +class STORE_GLOBALS { +public: + dlink link; /* double link */ + const char *name; /* resource name */ + int32_t NumConcurrentJobs; /* number of concurrent jobs running */ + int32_t NumConcurrentReadJobs; /* number of concurrent read jobs running */ + bool enabled; /* Enabled/disabled */ +}; + + /* * Store Resource * @@ -291,7 +318,7 @@ inline char *CLIENT::name() const { return hdr.name; } class STORE { public: RES hdr; - + STORE_GLOBALS *globals; /* global variables */ uint32_t SDport; /* port where Directors connect */ uint32_t SDDport; /* data port for File daemon */ char *address; @@ -301,8 +328,6 @@ public: alist *device; /* Alternate devices for this Storage */ int32_t MaxConcurrentJobs; /* Maximum concurrent jobs */ int32_t MaxConcurrentReadJobs; /* Maximum concurrent jobs reading */ - int32_t NumConcurrentJobs; /* number of concurrent jobs running */ - int32_t NumConcurrentReadJobs; /* number of jobs reading */ char *tls_ca_certfile; /* TLS CA Certificate File */ char *tls_ca_certdir; /* TLS CA Certificate Directory */ char *tls_certfile; /* TLS Client Certificate File */ @@ -311,9 +336,12 @@ public: bool tls_authenticate; /* Authenticated with TLS */ bool tls_enable; /* Enable TLS */ bool tls_require; /* Require TLS */ - bool enabled; /* Set if device is enabled */ + bool Enabled; /* Set if device is enabled */ bool AllowCompress; /* set if this Storage should allow jobs to enable compression */ bool autochanger; /* set if we are part of an autochanger */ + POOLMEM *ac_group; /* Autochanger StorageId group */ + STORE *changer; /* points to autochanger */ + STORE *shared_storage; /* points to shared storage */ int64_t StorageId; /* Set from Storage DB record */ utime_t heartbeat_interval; /* Interval to send heartbeats */ uint32_t drives; /* number of drives in autochanger */ @@ -321,6 +349,13 @@ public: /* Methods */ char *dev_name() const; char *name() const; + void create_store_globals(); + int32_t getNumConcurrentJobs(); + int32_t getNumConcurrentReadJobs(); + void setNumConcurrentJobs(int32_t num); + void setNumConcurrentReadJobs(int32_t num); + bool is_enabled(); + void setEnabled(bool val); }; inline char *STORE::dev_name() const @@ -366,6 +401,13 @@ inline void USTORE::set_source(const char *where) pm_strcpy(store_source, where); } +class JOB_GLOBALS { +public: + dlink link; /* double link */ + const char *name; /* resource name */ + int32_t NumConcurrentJobs; /* number of concurrent jobs running */ + bool enabled; /* Enabled/disabled */ +}; /* * Job Resource @@ -373,7 +415,7 @@ inline void USTORE::set_source(const char *where) class JOB { public: RES hdr; - + JOB_GLOBALS *globals; /* global variables */ uint32_t JobType; /* job type (backup, verify, restore */ uint32_t JobLevel; /* default backup/verify level */ uint32_t RestoreJobId; /* What -- JobId to restore */ @@ -409,8 +451,8 @@ public: utime_t SnapRetention; /* Snapshot retention period in seconds */ int64_t spool_size; /* Size of spool file for this job */ int32_t MaxConcurrentJobs; /* Maximum concurrent jobs */ - int32_t NumConcurrentJobs; /* number of concurrent jobs running */ uint32_t MaxSpawnedJobs; /* Max Jobs that can be started by Migration/Copy */ + uint32_t BackupsToKeep; /* Number of backups to keep in Virtual Full */ bool allow_mixed_priority; /* Allow jobs with higher priority concurrently with this */ MSGS *messages; /* How and where to send messages */ @@ -444,7 +486,7 @@ public: bool rerun_failed_levels; /* Upgrade to rerun failed levels */ bool PreferMountedVolumes; /* Prefer vols mounted rather than new one */ bool write_part_after_job; /* Set to write part after job in SD */ - bool enabled; /* Set if job enabled */ + bool Enabled; /* Set if job enabled */ bool accurate; /* Set if it is an accurate backup job */ bool AllowDuplicateJobs; /* Allow duplicate jobs */ bool AllowHigherDuplicates; /* Permit Higher Level */ @@ -452,13 +494,18 @@ public: bool CancelQueuedDuplicates; /* Cancel queued jobs */ bool CancelRunningDuplicates; /* Cancel Running jobs */ bool PurgeMigrateJob; /* Purges source job on completion */ - bool IgnoreDuplicateJobChecking; /* Set to ignore Duplicate Job Checking */ + bool DeleteConsolidatedJobs; /* Delete or not consolidated Virtual Full jobs */ alist *base; /* Base jobs */ int64_t max_bandwidth; /* Speed limit on this job */ /* Methods */ char *name() const; + void create_job_globals(); + int32_t getNumConcurrentJobs(); + void setNumConcurrentJobs(int32_t num); + bool is_enabled(); + void setEnabled(bool val); }; inline char *JOB::name() const { return hdr.name; } @@ -491,6 +538,7 @@ enum { INC_KW_STRIPPATH, INC_KW_HONOR_NODUMP, INC_KW_XATTR, + INC_KW_DEDUP, INC_KW_MAX /* Keep this last */ }; @@ -554,18 +602,28 @@ public: inline char *FILESET::name() const { return hdr.name; } +class SCHED_GLOBALS { +public: + dlink link; /* double link */ + const char *name; /* resource name */ + bool enabled; /* Enabled/disabled */ +}; + /* * Schedule Resource - * */ class SCHED { public: RES hdr; - + SCHED_GLOBALS *globals; RUN *run; - bool enabled; /* set if enabled */ + bool Enabled; /* set if enabled */ + /* Methods */ char *name() const; + void create_sched_globals(); + bool is_enabled(); + void setEnabled(bool val); }; inline char *SCHED::name() const { return hdr.name; } @@ -583,6 +641,7 @@ public: COUNTER *WrapCounter; /* Wrap counter name */ CAT *Catalog; /* Where to store */ bool created; /* Created in DB */ + /* Methods */ char *name() const; }; @@ -603,6 +662,7 @@ public: int32_t LabelType; /* Bacula/ANSI/IBM label type */ uint32_t max_volumes; /* max number of volumes */ utime_t VolRetention; /* volume retention period in seconds */ + utime_t CacheRetention; /* cloud cache retention period in seconds */ utime_t VolUseDuration; /* duration volume can be used */ uint32_t MaxVolJobs; /* Maximum jobs on the Volume */ uint32_t MaxVolFiles; /* Maximum files on the Volume */ @@ -699,6 +759,7 @@ public: #define GetPoolResWithName(x) ((POOL *)GetResWithName(R_POOL, (x))) #define GetStoreResWithName(x) ((STORE *)GetResWithName(R_STORAGE, (x))) +#define GetSchedResWithName(x) ((SCHED *)GetResWithName(R_SCHEDULE, (x))) #define GetClientResWithName(x) ((CLIENT *)GetResWithName(R_CLIENT, (x))) #define GetJobResWithName(x) ((JOB *)GetResWithName(R_JOB, (x))) #define GetFileSetResWithName(x) ((FILESET *)GetResWithName(R_FILESET, (x))) @@ -711,6 +772,7 @@ void store_replace(LEX *lc, RES_ITEM *item, int index, int pass); void store_migtype(LEX *lc, RES_ITEM *item, int index, int pass); void store_acl(LEX *lc, RES_ITEM *item, int index, int pass); void store_ac_res(LEX *lc, RES_ITEM *item, int index, int pass); +void store_device(LEX *lc, RES_ITEM *item, int index, int pass); void store_actiononpurge(LEX *lc, RES_ITEM *item, int index, int pass); void store_inc(LEX *lc, RES_ITEM *item, int index, int pass); void store_regex(LEX *lc, RES_ITEM *item, int index, int pass); diff --git a/bacula/src/dird/expand.c b/bacula/src/dird/expand.c index 367f78570d..b1e7707de6 100644 --- a/bacula/src/dird/expand.c +++ b/bacula/src/dird/expand.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -22,7 +22,6 @@ * in particular for the LabelFormat specification. * * Kern Sibbald, June MMIII - * */ #include "bacula.h" diff --git a/bacula/src/dird/fd_cmds.c b/bacula/src/dird/fd_cmds.c index efb5596dc6..e6fd2930a2 100644 --- a/bacula/src/dird/fd_cmds.c +++ b/bacula/src/dird/fd_cmds.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -101,7 +101,7 @@ int connect_to_file_daemon(JCR *jcr, int retry_interval, int max_retry_time, bstrncat(name, jcr->client->name(), sizeof(name)); fd->set_source_address(director->DIRsrc_addr); - if (!fd->connect(jcr,retry_interval,max_retry_time, heart_beat, name, jcr->client->address, + if (!fd->connect(jcr,retry_interval,max_retry_time, heart_beat, name, jcr->client->address(), NULL, jcr->client->FDport, verbose)) { fd->close(); jcr->setJobStatus(JS_ErrorTerminated); @@ -275,16 +275,16 @@ void get_level_since_time(JCR *jcr, char *since, int since_len) /* Get the end time of our most recent successfull backup for this job */ /* This will be used to see if there have been any failures since then */ - if (db_find_last_job_end_time(jcr, jcr->db, &jcr->jr, &etime, prev_job)) { + if (db_find_last_job_end_time(jcr, jcr->db, &jcr->jr, &etime, prev_job)) { - /* See if there are any failed Differential/Full backups since the completion */ + /* See if there are any failed Differential/Full backups since the completion */ /* of our last successful backup for this job */ if (db_find_failed_job_since(jcr, jcr->db, &jcr->jr, etime, JobLevel)) { /* If our job is an Incremental and we have a failed job then upgrade. */ /* If our job is a Differential and the failed job is a Full then upgrade. */ /* Otherwise there is no reason to upgrade. */ - if ((jcr->getJobLevel() == L_INCREMENTAL) || + if ((jcr->getJobLevel() == L_INCREMENTAL) || ((jcr->getJobLevel() == L_DIFFERENTIAL) && (JobLevel == L_FULL))) { Jmsg(jcr, M_INFO, 0, _("Prior failed job found in catalog. Upgrading to %s.\n"), level_to_str(JobLevel)); @@ -574,7 +574,7 @@ static bool send_list_item(JCR *jcr, const char *code, char *item, BSOCK *fd) break; case '<': p++; /* skip over < */ - if ((ffd = fopen(p, "rb")) == NULL) { + if ((ffd = bfopen(p, "rb")) == NULL) { berrno be; Jmsg(jcr, M_FATAL, 0, _("Cannot open included file: %s. ERR=%s\n"), p, be.bstrerror()); @@ -625,6 +625,30 @@ bool send_include_list(JCR *jcr) return true; } +/* + * + * Send a include list with only one directory and recurse=no + * TODO: Need to display the plugin somewhere + * The main point is that we don't introduce any protocol change + */ +bool send_ls_fileset(JCR *jcr, const char *path) +{ + BSOCK *fd = jcr->file_bsock; + fd->fsend(filesetcmd, "" /* no vss */, "" /* no snapshot */); + + fd->fsend("I\n"); + fd->fsend("O h\n"); /* Limit recursion to one directory */ + fd->fsend("N\n"); + fd->fsend("F %s\n", path); + fd->fsend("N\n"); + fd->signal(BNET_EOD); /* end of data */ + + if (!response(jcr, fd, OKinc, "Include", DISPLAY_ERROR)) { + return false; + } + return true; +} + /* * Send exclude list to File daemon * Under the new scheme, the Exclude list diff --git a/bacula/src/dird/getmsg.c b/bacula/src/dird/getmsg.c index a9d308407a..07fd816e59 100644 --- a/bacula/src/dird/getmsg.c +++ b/bacula/src/dird/getmsg.c @@ -44,9 +44,9 @@ /* Forward referenced functions */ static char *find_msg_start(char *msg); -static char Job_status[] = "Status Job=%127s JobStatus=%d\n"; +static char Job_status[] = "Status JobId=%ld JobStatus=%d\n"; #ifdef needed -static char Device_update[] = "DevUpd Job=%127s " +static char Device_update[] = "DevUpd JobId=%127s " "device=%127s " "append=%d read=%d num_writers=%d " "open=%d labeled=%d offline=%d " @@ -91,6 +91,23 @@ static void set_jcr_sd_job_status(JCR *jcr, int SDJobStatus) } +/* + * See if we are pointing to a message id + * Look for: [XYnnnn] + */ +static bool is_msgid(char *msg) +{ + if (!msg) return false; + char *end = strchr(msg, ']'); + if (!end) return false; + if ((end - msg) != 7) return false; + if (!B_ISUPPER(msg[1]) || !B_ISUPPER(msg[2])) return false; + for (int i=3; i<7; i++) { + if (!B_ISDIGIT(msg[i])) return false; + } + return true; +} + /* * Get a message * Call appropriate processing routine @@ -108,7 +125,8 @@ static void set_jcr_sd_job_status(JCR *jcr, int SDJobStatus) * All other messages are expected begin with some identifier * -- for the moment only the first character is checked, but * at a later time, the whole identifier (e.g. Jmsg, CatReq, ...) - * could be checked. This is followed by Job=Jobname + * could be checked. + * This is followed by JobId=nnn * info. The identifier is used to dispatch the message to the right * place (Job message, catalog request, ...). The Job is used to lookup * the JCR so that the action is performed on the correct jcr, and @@ -120,7 +138,7 @@ static void set_jcr_sd_job_status(JCR *jcr, int SDJobStatus) int bget_dirmsg(BSOCK *bs) { int32_t n = BNET_TERMINATE; - char Job[MAX_NAME_LENGTH]; + JobId_t JobId = 0; char MsgType[20]; int type; utime_t mtime; /* message time */ @@ -181,12 +199,15 @@ int bget_dirmsg(BSOCK *bs) * a message to dispatch, or a catalog request. * Try to fulfill it. */ - if (sscanf(bs->msg, "%020s Job=%127s ", MsgType, Job) != 2) { + if (sscanf(bs->msg, "%020s JobId=%ld ", MsgType, &JobId) != 2) { + if (is_msgid(strchr(bs->msg, '['))) { + return n; + } Jmsg1(jcr, M_ERROR, 0, _("Malformed message: %s\n"), bs->msg); continue; } - /* Skip past "Jmsg Job=nnn" */ + /* Skip past "Jmsg JobId=nnn" */ if (!(msg=find_msg_start(bs->msg))) { Jmsg1(jcr, M_ERROR, 0, _("Malformed message: %s\n"), bs->msg); continue; @@ -194,13 +215,13 @@ int bget_dirmsg(BSOCK *bs) /* * Here we are expecting a message of the following format: - * Jmsg Job=nnn type=nnn level=nnn Message-string + * Jmsg JobId=nnn type=nnn level=nnn Message-string * Note, level should really be mtime, but that changes * the protocol. */ if (bs->msg[0] == 'J') { /* Job message */ - if (sscanf(bs->msg, "Jmsg Job=%127s type=%d level=%lld", - Job, &type, &mtime) != 3) { + if (sscanf(bs->msg, "Jmsg JobId=%ld type=%d level=%lld", + &JobId, &type, &mtime) != 3) { Jmsg1(jcr, M_ERROR, 0, _("Malformed message: %s\n"), bs->msg); continue; } @@ -218,7 +239,7 @@ int bget_dirmsg(BSOCK *bs) } /* * Here we expact a CatReq message - * CatReq Job=nn Catalog-Request-Message + * CatReq JobId=nn Catalog-Request-Message */ if (bs->msg[0] == 'C') { /* Catalog request */ Dmsg2(900, "Catalog req jcr=%p: %s", jcr, bs->msg); @@ -233,8 +254,8 @@ int bget_dirmsg(BSOCK *bs) if (bs->msg[0] == 'B') { /* SD sending file spool attributes */ Dmsg2(100, "Blast attributes jcr=%p: %s", jcr, bs->msg); char filename[256]; - if (sscanf(bs->msg, "BlastAttr Job=%127s File=%255s", - Job, filename) != 2) { + if (sscanf(bs->msg, "BlastAttr JobId=%ld File=%255s", + &JobId, filename) != 2) { Jmsg1(jcr, M_ERROR, 0, _("Malformed message: %s\n"), bs->msg); continue; } @@ -267,8 +288,7 @@ int bget_dirmsg(BSOCK *bs) } if (bs->msg[0] == 'S') { /* Status change */ int JobStatus; - char Job[MAX_NAME_LENGTH]; - if (sscanf(bs->msg, Job_status, &Job, &JobStatus) == 2) { + if (sscanf(bs->msg, Job_status, &JobId, &JobStatus) == 2) { set_jcr_sd_job_status(jcr, JobStatus); /* current status */ } else { Jmsg1(jcr, M_ERROR, 0, _("Malformed message: %s\n"), bs->msg); diff --git a/bacula/src/dird/job.c b/bacula/src/dird/job.c index 60c70f18ad..3ac5feb645 100644 --- a/bacula/src/dird/job.c +++ b/bacula/src/dird/job.c @@ -11,17 +11,15 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director Job processing routines * * Kern Sibbald, October MM - * */ #include "bacula.h" @@ -588,51 +586,64 @@ static bool cancel_sd_job(UAContext *ua, const char *cmd, JCR *jcr) /* The FD is not connected, so we try to complete JCR fields and send * the cancel command. */ -static int cancel_inactive_job(UAContext *ua, JCR *jcr) +int cancel_inactive_job(UAContext *ua) { CLIENT_DBR cr; JOB_DBR jr; int i; USTORE store; - CLIENT *client; + CLIENT *client; + JCR *jcr = new_jcr(sizeof(JCR), dird_free_jcr); - Dmsg2(10, "cancel_inactive_job ua.jcr=%p jcr=%p\n", ua->jcr, jcr); + memset(&jr, 0, sizeof(jr)); + memset(&cr, 0, sizeof(cr)); - if (!jcr->client) { - memset(&cr, 0, sizeof(cr)); + if ((i = find_arg_with_value(ua, "jobid")) > 0) { + jr.JobId = str_to_int64(ua->argv[i]); - /* User is kind enough to provide the client name */ - if ((i = find_arg_with_value(ua, "client")) > 0) { - bstrncpy(cr.Name, ua->argv[i], sizeof(cr.Name)); - } else { - memset(&jr, 0, sizeof(jr)); - bstrncpy(jr.Job, jcr->Job, sizeof(jr.Job)); + } else if ((i = find_arg_with_value(ua, "ujobid")) > 0) { + bstrncpy(jr.Job, ua->argv[i], sizeof(jr.Job)); - if (!open_client_db(ua)) { - goto bail_out; - } - if (!db_get_job_record(ua->jcr, ua->db, &jr)) { - goto bail_out; - } - cr.ClientId = jr.ClientId; - if (!cr.ClientId || !db_get_client_record(ua->jcr, ua->db, &cr)) { - goto bail_out; - } - } + } else { + ua->error_msg(_("jobid/ujobid argument not found.\n")); + goto bail_out; + } - if (acl_access_ok(ua, Client_ACL, cr.Name)) { - client = (CLIENT *)GetResWithName(R_CLIENT, cr.Name); - if (client) { - jcr->client = client; - } else { - Jmsg1(jcr, M_FATAL, 0, _("Client resource \"%s\" does not exist.\n"), cr.Name); - goto bail_out; - } + if (!open_client_db(ua)) { + goto bail_out; + } + + if (!db_get_job_record(ua->jcr, ua->db, &jr)) { + ua->error_msg(_("Job %ld/%s not found in database.\n"), jr.JobId, jr.Job); + goto bail_out; + } + + if (!acl_access_ok(ua, Job_ACL, jr.Name)) { + ua->error_msg(_("Job %s is not accessible from this console\n"), jr.Name); + goto bail_out; + } + + cr.ClientId = jr.ClientId; + if (!cr.ClientId || !db_get_client_record(ua->jcr, ua->db, &cr)) { + ua->error_msg(_("Client %ld not found in database.\n"), jr.ClientId); + goto bail_out; + } + + if (acl_access_client_ok(ua, cr.Name, jr.JobType)) { + client = (CLIENT *)GetResWithName(R_CLIENT, cr.Name); + if (client) { + jcr->client = client; } else { + Jmsg1(jcr, M_FATAL, 0, _("Client resource \"%s\" does not exist.\n"), cr.Name); goto bail_out; } + } else { + goto bail_out; } + jcr->JobId = jr.JobId; + bstrncpy(jcr->Job, jr.Job, sizeof(jcr->Job)); + cancel_file_daemon_job(ua, "cancel", jcr); /* At this time, we can't really guess the storage name from @@ -643,8 +654,7 @@ static int cancel_inactive_job(UAContext *ua, JCR *jcr) goto bail_out; } - set_wstorage(ua->jcr, &store); - + set_wstorage(jcr, &store); cancel_sd_job(ua, "cancel", jcr); bail_out: @@ -661,23 +671,15 @@ bail_out: * false on failure. Message sent to ua->jcr. */ bool -cancel_job(UAContext *ua, JCR *jcr, bool cancel) +cancel_job(UAContext *ua, JCR *jcr, int wait, bool cancel) { char ed1[50]; int32_t old_status = jcr->JobStatus; int status; const char *reason, *cmd; - bool force = find_arg(ua, "inactive") > 0; Dmsg3(10, "cancel_job jcr=%p jobid=%d use_count\n", jcr, jcr->JobId, jcr->use_count()); - /* If the user explicitely ask, we can send the cancel command to - * the FD. - */ - if (cancel && force) { - return cancel_inactive_job(ua, jcr); - } - if (cancel) { status = JS_Canceled; reason = _("canceled"); @@ -710,8 +712,11 @@ cancel_job(UAContext *ua, JCR *jcr, bool cancel) /* Cancel File daemon */ if (jcr->file_bsock) { + btimer_t *tid; /* do not return now, we want to try to cancel the sd */ + tid = start_bsock_timer(jcr->file_bsock, 120); cancel_file_daemon_job(ua, cmd, jcr); + stop_bsock_timer(tid); } /* We test file_bsock because the previous operation can take @@ -724,8 +729,11 @@ cancel_job(UAContext *ua, JCR *jcr, bool cancel) /* Cancel Storage daemon */ if (jcr->store_bsock) { + btimer_t *tid; /* do not return now, we want to try to cancel the sd socket */ + tid = start_bsock_timer(jcr->store_bsock, 120); cancel_sd_job(ua, cmd, jcr); + stop_bsock_timer(tid); } /* We test file_bsock because the previous operation can take @@ -744,10 +752,13 @@ cancel_job(UAContext *ua, JCR *jcr, bool cancel) JCR *wjcr = jcr->wjcr; if (wjcr->store_bsock) { + btimer_t *tid; /* do not return now, we want to try to cancel the sd socket */ + tid = start_bsock_timer(wjcr->store_bsock, 120); cancel_sd_job(ua, cmd, wjcr); + stop_bsock_timer(tid); } - /* We test file_bsock because the previous operation can take + /* We test store_bsock because the previous operation can take * several minutes */ if (wjcr->store_bsock && cancel) { @@ -819,14 +830,33 @@ static void job_monitor_destructor(watchdog_t *self) free_jcr(control_jcr); } -static void job_monitor_watchdog(watchdog_t *self) +extern "C" void *cancel_thread(void *arg) { - JCR *control_jcr, *jcr; + JCR *jcr = (JCR *)arg; + UAContext *ua; + JCR *control_jcr; + + pthread_detach(pthread_self()); + ua = new_ua_context(jcr); + control_jcr = new_control_jcr("*CancelThread*", JT_SYSTEM); + ua->jcr = control_jcr; + + Dmsg3(400, "Cancelling JCR %p JobId=%d (%s)\n", jcr, jcr->JobId, jcr->Job); + cancel_job(ua, jcr, 120); + Dmsg2(400, "Have cancelled JCR %p JobId=%d\n", jcr, jcr->JobId); + + free_ua_context(ua); + free_jcr(control_jcr); + free_jcr(jcr); + return NULL; +} - control_jcr = (JCR *)self->data; +static void job_monitor_watchdog(watchdog_t *wd) +{ + JCR *jcr; Dsm_check(100); - Dmsg1(800, "job_monitor_watchdog %p called\n", self); + Dmsg1(800, "job_monitor_watchdog %p called\n", wd); foreach_jcr(jcr) { bool cancel = false; @@ -854,14 +884,15 @@ static void job_monitor_watchdog(watchdog_t *self) } if (cancel) { - Dmsg3(800, "Cancelling JCR %p jobid %d (%s)\n", jcr, jcr->JobId, jcr->Job); - UAContext *ua = new_ua_context(jcr); - ua->jcr = control_jcr; - cancel_job(ua, jcr); - free_ua_context(ua); - Dmsg2(800, "Have cancelled JCR %p Job=%d\n", jcr, jcr->JobId); + pthread_t thid; + int status; + jcr->inc_use_count(); + if ((status=pthread_create(&thid, NULL, cancel_thread, (void *)jcr)) != 0) { + berrno be; + Jmsg1(jcr, M_WARNING, 0, _("Cannot create cancel thread: ERR=%s\n"), be.bstrerror(status)); + free_jcr(jcr); + } } - } /* Keep reference counts correct */ endeach_jcr(jcr); @@ -991,7 +1022,7 @@ bool allow_duplicate_job(JCR *jcr) JOB *job = jcr->job; JCR *djcr; /* possible duplicate job */ - /* Is AllowDuplicateJobs is set or is duplicate checking + /* Is AllowDuplicateJobs is set or is duplicate checking * disabled for this job? */ if (job->AllowDuplicateJobs || jcr->IgnoreDuplicateJobChecking) { return true; @@ -1003,14 +1034,14 @@ bool allow_duplicate_job(JCR *jcr) */ foreach_jcr(djcr) { - if (jcr == djcr || djcr->JobId == 0) { + if (jcr == djcr || djcr->is_internal_job() || !djcr->job) { continue; /* do not cancel this job or consoles */ } /* Does Job has the IgnoreDuplicateJobChecking flag set, * if so do not check it against other jobs */ if (djcr->IgnoreDuplicateJobChecking) { - continue; - } + continue; + } if ((strcmp(job->name(), djcr->job->name()) == 0) && djcr->getJobType() == jcr->getJobType()) /* A duplicate is about the same name and the same type */ { @@ -1055,7 +1086,7 @@ bool allow_duplicate_job(JCR *jcr) djcr->JobId); break; /* get out of foreach_jcr */ } - } + } /* Cancel one of the two jobs (me or dup) */ /* If CancelQueuedDuplicates is set do so only if job is queued */ if (job->CancelQueuedDuplicates) { @@ -1078,10 +1109,10 @@ bool allow_duplicate_job(JCR *jcr) /* Zap the duplicated job djcr */ UAContext *ua = new_ua_context(jcr); Jmsg(jcr, M_INFO, 0, _("Cancelling duplicate JobId=%d.\n"), djcr->JobId); - cancel_job(ua, djcr); + cancel_job(ua, djcr, 60); bmicrosleep(0, 500000); djcr->setJobStatus(JS_Canceled); - cancel_job(ua, djcr); + cancel_job(ua, djcr, 60); free_ua_context(ua); Dmsg2(800, "Cancel dup %p JobId=%d\n", djcr, djcr->JobId); } else { @@ -1179,17 +1210,6 @@ void apply_pool_overrides(JCR *jcr) } } break; - case L_VIRTUAL_FULL: - if (jcr->vfull_pool) { - jcr->pool = jcr->vfull_pool; - pool_override = true; - if (jcr->run_vfull_pool_override) { - pm_strcpy(jcr->pool_source, _("Run VFullPool override")); - } else { - pm_strcpy(jcr->pool_source, _("Job VFullPool override")); - } - } - break; case L_INCREMENTAL: if (jcr->inc_pool) { jcr->pool = jcr->inc_pool; @@ -1323,7 +1343,7 @@ void update_job_end_record(JCR *jcr) jcr->jr.ReadBytes = jcr->ReadBytes; jcr->jr.VolSessionId = jcr->VolSessionId; jcr->jr.VolSessionTime = jcr->VolSessionTime; - jcr->jr.JobErrors = jcr->JobErrors; + jcr->jr.JobErrors = jcr->JobErrors + jcr->SDErrors; jcr->jr.HasBase = jcr->HasBase; if (!db_update_job_end_record(jcr, jcr->db, &jcr->jr)) { Jmsg(jcr, M_WARNING, 0, _("Error updating job record. %s"), @@ -1455,16 +1475,21 @@ void dird_free_jcr(JCR *jcr) free_and_null_pool_memory(jcr->wstore_source); free_and_null_pool_memory(jcr->rstore_source); free_and_null_pool_memory(jcr->next_vol_list); + free_and_null_pool_memory(jcr->component_fname); /* Delete lists setup to hold storage pointers */ free_rwstorage(jcr); jcr->job_end_push.destroy(); - if (jcr->JobId != 0) { + if (jcr->JobId != 0) write_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - } + if (jcr->plugin_config) { + free_plugin_config_items(jcr->plugin_config); + delete jcr->plugin_config; + jcr->plugin_config = NULL; + } free_plugins(jcr); /* release instantiated plugins */ garbage_collect_memory_pool(); @@ -1560,7 +1585,6 @@ void set_jcr_defaults(JCR *jcr, JOB *job) pm_strcpy(jcr->next_pool_source, _("Job Pool's NextPool resource")); } jcr->full_pool = job->full_pool; - jcr->vfull_pool = job->vfull_pool; jcr->inc_pool = job->inc_pool; jcr->diff_pool = job->diff_pool; if (job->pool->catalog) { @@ -1576,7 +1600,6 @@ void set_jcr_defaults(JCR *jcr, JOB *job) jcr->spool_data = job->spool_data; jcr->spool_size = job->spool_size; jcr->write_part_after_job = job->write_part_after_job; - jcr->IgnoreDuplicateJobChecking = job->IgnoreDuplicateJobChecking; jcr->MaxRunSchedTime = job->MaxRunSchedTime; if (jcr->RestoreBootstrap) { free(jcr->RestoreBootstrap); @@ -1797,45 +1820,55 @@ void create_clones(JCR *jcr) } /* - * Given: a JobId in jcr->previous_jr.JobId, + * Given: a JobId and FileIndex * this subroutine writes a bsr file to restore that job. * Returns: -1 on error * number of files if OK */ -int create_restore_bootstrap_file(JCR *jcr) +int create_restore_bootstrap_file(JCR *jcr, JobId_t jobid, int findex1, int findex2) { RESTORE_CTX rx; UAContext *ua; int files; memset(&rx, 0, sizeof(rx)); - rx.bsr = new_bsr(); rx.JobIds = (char *)""; - rx.bsr->JobId = jcr->previous_jr.JobId; + + rx.bsr_list = create_bsr_list(jobid, findex1, findex2); + ua = new_ua_context(jcr); - if (!complete_bsr(ua, rx.bsr)) { + if (!complete_bsr(ua, rx.bsr_list)) { files = -1; goto bail_out; } - rx.bsr->fi = new_findex(); - rx.bsr->fi->findex = 1; - rx.bsr->fi->findex2 = jcr->previous_jr.JobFiles; + jcr->ExpectedFiles = write_bsr_file(ua, rx); if (jcr->ExpectedFiles == 0) { files = 0; goto bail_out; } free_ua_context(ua); - free_bsr(rx.bsr); + free_bsr(rx.bsr_list); jcr->needs_sd = true; return jcr->ExpectedFiles; bail_out: free_ua_context(ua); - free_bsr(rx.bsr); + free_bsr(rx.bsr_list); return files; } +/* + * Given: a JobId in jcr->previous_jr.JobId, + * this subroutine writes a bsr file to restore that job. + * Returns: -1 on error + * number of files if OK + */ +int create_restore_bootstrap_file(JCR *jcr) +{ + return create_restore_bootstrap_file(jcr, jcr->previous_jr.JobId, 1, jcr->previous_jr.JobFiles); +} + /* TODO: redirect command ouput to job log */ bool run_console_command(JCR *jcr, const char *cmd) { @@ -1853,6 +1886,7 @@ bool run_console_command(JCR *jcr, const char *cmd) } else { ok = do_a_command(ua); } + close_db(ua); free_ua_context(ua); free_jcr(ljcr); return ok; diff --git a/bacula/src/dird/jobq.c b/bacula/src/dird/jobq.c index 95168218b4..6363d9a98d 100644 --- a/bacula/src/dird/jobq.c +++ b/bacula/src/dird/jobq.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -153,7 +153,7 @@ struct wait_pkt { * this routine is only used for jobs started from the console * for which the user explicitly specified a start time. Otherwise * most jobs are put into the job queue only when their - * scheduled time arives. + * scheduled time arrives. */ extern "C" void *sched_wait(void *arg) @@ -191,9 +191,10 @@ void *sched_wait(void *arg) return NULL; } -/* Procedure to update the Client->NumConcurrentJobs */ +/* Procedure to update the client->NumConcurrentJobs */ static void update_client_numconcurrentjobs(JCR *jcr, int val) { + int num; if (!jcr->client) { return; } @@ -205,12 +206,13 @@ static void update_client_numconcurrentjobs(JCR *jcr, int val) case JT_ADMIN: break; case JT_BACKUP: - if (jcr->no_client_used()) { + /* Fall through wanted */ + default: + if (jcr->no_client_used() || jcr->wasVirtualFull) { break; } - /* Failback wanted */ - default: - jcr->client->NumConcurrentJobs += val; + num = jcr->client->getNumConcurrentJobs(); + jcr->client->setNumConcurrentJobs(num + val); break; } } @@ -482,10 +484,12 @@ void *jobq_server(void *arg) * put into the ready queue. */ if (jcr->acquired_resource_locks) { + int num; dec_read_store(jcr); dec_write_store(jcr); update_client_numconcurrentjobs(jcr, -1); - jcr->job->NumConcurrentJobs--; + num = jcr->job->getNumConcurrentJobs() - 1; + jcr->job->setNumConcurrentJobs(num); jcr->acquired_resource_locks = false; } @@ -635,16 +639,18 @@ static bool reschedule_job(JCR *jcr, jobq_t *jq, jobq_item_t *je) /* Basic condition is that more reschedule times remain */ if (jcr->job->RescheduleTimes == 0 || jcr->reschedule_count < jcr->job->RescheduleTimes) { - resched = - /* Check for incomplete jobs */ - (jcr->RescheduleIncompleteJobs && - jcr->is_incomplete() && jcr->is_JobType(JT_BACKUP) && - !(jcr->HasBase||jcr->is_JobLevel(L_BASE))) || + + /* Check for incomplete jobs */ + if (jcr->is_incomplete()) { + resched = (jcr->RescheduleIncompleteJobs && jcr->is_JobType(JT_BACKUP) && + !(jcr->HasBase||jcr->is_JobLevel(L_BASE))); + } else { /* Check for failed jobs */ - (jcr->job->RescheduleOnError && - !jcr->is_JobStatus(JS_Terminated) && - !jcr->is_JobStatus(JS_Canceled) && - jcr->is_JobType(JT_BACKUP)); + resched = (jcr->job->RescheduleOnError && + !jcr->is_JobStatus(JS_Terminated) && + !jcr->is_JobStatus(JS_Canceled) && + jcr->is_JobType(JT_BACKUP)); + } } if (resched) { char dt[50], dt2[50]; @@ -794,7 +800,7 @@ static bool acquire_resources(JCR *jcr) if (jcr->rstore) { Dmsg1(200, "Rstore=%s\n", jcr->rstore->name()); if (!inc_read_store(jcr)) { - Dmsg1(200, "Fail rncj=%d\n", jcr->rstore->NumConcurrentJobs); + Dmsg1(200, "Fail rncj=%d\n", jcr->rstore->getNumConcurrentJobs()); jcr->setJobStatus(JS_WaitStoreRes); return false; } @@ -802,14 +808,15 @@ static bool acquire_resources(JCR *jcr) if (jcr->wstore) { Dmsg1(200, "Wstore=%s\n", jcr->wstore->name()); - if (jcr->wstore->NumConcurrentJobs < jcr->wstore->MaxConcurrentJobs) { - jcr->wstore->NumConcurrentJobs++; - Dmsg1(200, "Inc wncj=%d\n", jcr->wstore->NumConcurrentJobs); + int num = jcr->wstore->getNumConcurrentJobs(); + if (num < jcr->wstore->MaxConcurrentJobs) { + Dmsg1(200, "Inc wncj=%d\n", num + 1); + jcr->wstore->setNumConcurrentJobs(num + 1); } else if (jcr->rstore) { dec_read_store(jcr); skip_this_jcr = true; } else { - Dmsg1(200, "Fail wncj=%d\n", jcr->wstore->NumConcurrentJobs); + Dmsg1(200, "Fail wncj=%d\n", num); skip_this_jcr = true; } } @@ -819,7 +826,7 @@ static bool acquire_resources(JCR *jcr) } if (jcr->client) { - if (jcr->client->NumConcurrentJobs < jcr->client->MaxConcurrentJobs) { + if (jcr->client->getNumConcurrentJobs() < jcr->client->MaxConcurrentJobs) { update_client_numconcurrentjobs(jcr, 1); } else { /* Back out previous locks */ @@ -829,8 +836,10 @@ static bool acquire_resources(JCR *jcr) return false; } } - if (jcr->job->NumConcurrentJobs < jcr->job->MaxConcurrentJobs) { - jcr->job->NumConcurrentJobs++; + if (jcr->job->getNumConcurrentJobs() < jcr->job->MaxConcurrentJobs) { + int num; + num = jcr->job->getNumConcurrentJobs() + 1; + jcr->job->setNumConcurrentJobs(num); } else { /* Back out previous locks */ dec_write_store(jcr); @@ -853,13 +862,17 @@ static pthread_mutex_t rstore_mutex = PTHREAD_MUTEX_INITIALIZER; bool inc_read_store(JCR *jcr) { P(rstore_mutex); - if (jcr->rstore->NumConcurrentJobs < jcr->rstore->MaxConcurrentJobs && + int num = jcr->rstore->getNumConcurrentJobs(); + int numread = jcr->rstore->getNumConcurrentReadJobs(); + if (num < jcr->rstore->MaxConcurrentJobs && (jcr->getJobType() == JT_RESTORE || - jcr->rstore->MaxConcurrentReadJobs == 0 || - jcr->rstore->NumConcurrentReadJobs < jcr->rstore->MaxConcurrentReadJobs)) { - jcr->rstore->NumConcurrentReadJobs++; - jcr->rstore->NumConcurrentJobs++; - Dmsg1(200, "Inc rncj=%d\n", jcr->rstore->NumConcurrentJobs); + numread == 0 || + numread < jcr->rstore->MaxConcurrentReadJobs)) { + num++; + numread++; + jcr->rstore->setNumConcurrentReadJobs(numread); + jcr->rstore->setNumConcurrentJobs(num); + Dmsg1(200, "Inc rncj=%d\n", num); V(rstore_mutex); return true; } @@ -871,20 +884,20 @@ void dec_read_store(JCR *jcr) { if (jcr->rstore) { P(rstore_mutex); - jcr->rstore->NumConcurrentReadJobs--; /* back out rstore */ - jcr->rstore->NumConcurrentJobs--; /* back out rstore */ - Dmsg1(200, "Dec rncj=%d\n", jcr->rstore->NumConcurrentJobs); + int numread = jcr->rstore->getNumConcurrentReadJobs() - 1; + int num = jcr->rstore->getNumConcurrentJobs() - 1; + jcr->rstore->setNumConcurrentReadJobs(numread); + jcr->rstore->setNumConcurrentJobs(num); + Dmsg1(200, "Dec rncj=%d\n", num); V(rstore_mutex); - ASSERT(jcr->rstore->NumConcurrentReadJobs >= 0); - ASSERT(jcr->rstore->NumConcurrentJobs >= 0); } } static void dec_write_store(JCR *jcr) { if (jcr->wstore) { - jcr->wstore->NumConcurrentJobs--; - Dmsg1(200, "Dec wncj=%d\n", jcr->wstore->NumConcurrentJobs); - ASSERT(jcr->wstore->NumConcurrentJobs >= 0); + int num = jcr->wstore->getNumConcurrentJobs() - 1; + Dmsg1(200, "Dec wncj=%d\n", num); + jcr->wstore->setNumConcurrentJobs(num); } } diff --git a/bacula/src/dird/jobq.h b/bacula/src/dird/jobq.h index c365d269a4..ec32866913 100644 --- a/bacula/src/dird/jobq.h +++ b/bacula/src/dird/jobq.h @@ -1,18 +1,7 @@ -/* - * Bacula job queue routines. - * - * Kern Sibbald, July MMIII - * - * This code adapted from Bacula work queue code, which was - * adapted from "Programming with POSIX Threads", by - * David R. Butenhof - * - */ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -27,6 +16,15 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ +/* + * Bacula job queue routines. + * + * Kern Sibbald, July MMIII + * + * This code adapted from Bacula work queue code, which was + * adapted from "Programming with POSIX Threads", by + * David R. Butenhof + */ #ifndef __JOBQ_H #define __JOBQ_H 1 diff --git a/bacula/src/dird/mac.c b/bacula/src/dird/mac.c index 58d50a5d06..9d4b135c41 100644 --- a/bacula/src/dird/mac.c +++ b/bacula/src/dird/mac.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- mac.c -- responsible for doing * migration and copy jobs. * @@ -31,7 +30,6 @@ * Open connection with Storage daemon and pass him commands * to do the backup. * When the Storage daemon finishes the job, update the DB. - * */ #include "bacula.h" @@ -189,6 +187,12 @@ bool do_mac_init(JCR *jcr) if (wjcr->getJobLevel() == L_VIRTUAL_FULL) { wjcr->setJobLevel(L_INCREMENTAL); } + + /* Don't check for duplicates on this jobs. We do it before setup_job(), + * because we check allow_duplicate_job() here. + */ + wjcr->IgnoreDuplicateJobChecking = true; + if (!setup_job(wjcr)) { Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n")); return false; @@ -207,8 +211,6 @@ bool do_mac_init(JCR *jcr) /* Don't let WatchDog checks Max*Time value on this Job */ wjcr->no_maxtime = true; - /* Don't check for duplicates on this jobs */ - wjcr->job->IgnoreDuplicateJobChecking = true; Dmsg4(dbglevel, "wjcr: Name=%s JobId=%d Type=%c Level=%c\n", wjcr->jr.Name, (int)wjcr->jr.JobId, wjcr->jr.JobType, wjcr->jr.JobLevel); @@ -620,8 +622,9 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH]; char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50]; char ec6[50], ec7[50], ec8[50], ec9[30], ec10[30]; - char term_code[100], sd_term_msg[100]; - const char *term_msg; + char sd_term_msg[100]; + POOL_MEM term_code; + POOL_MEM term_msg; int msg_type = M_INFO; MEDIA_DBR mr; double kbps; @@ -631,6 +634,8 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) POOL_MEM query(PM_MESSAGE); POOL_MEM vol_info; + remove_dummy_jobmedia_records(jcr); + Dmsg2(100, "Enter mac_cleanup %d %c\n", TermCode, TermCode); update_job_end(jcr, TermCode); @@ -648,7 +653,7 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) wjcr->JobBytes = jcr->JobBytes = wjcr->SDJobBytes; wjcr->jr.RealEndTime = 0; wjcr->jr.PriorJobId = jcr->previous_jr.JobId; - + wjcr->JobErrors += wjcr->SDErrors; update_job_end(wjcr, TermCode); /* Update final items to set them to the previous job's values */ @@ -779,14 +784,14 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) switch (jcr->JobStatus) { case JS_Terminated: if (jcr->JobErrors || jcr->SDErrors) { - term_msg = _("%s OK -- with warnings"); + Mmsg(term_msg, _("%%s OK -- %s"), jcr->StatusErrMsg[0] ? jcr->StatusErrMsg : _("with warnings")); } else { - term_msg = _("%s OK"); + Mmsg(term_msg, _("%%s OK")); } break; case JS_FatalError: case JS_ErrorTerminated: - term_msg = _("*** %s Error ***"); + Mmsg(term_msg, _("*** %%s Error ***")); msg_type = M_ERROR; /* Generate error message */ if (jcr->store_bsock) { jcr->store_bsock->signal(BNET_TERMINATE); @@ -802,7 +807,7 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) } break; case JS_Canceled: - term_msg = _("%s Canceled"); + Mmsg(term_msg, _("%%s Canceled")); if (jcr->store_bsock) { jcr->store_bsock->signal(BNET_TERMINATE); if (jcr->SD_msg_chan_started) { @@ -817,7 +822,7 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) } break; default: - term_msg = _("Inappropriate %s term code"); + Mmsg(term_msg, _("Inappropriate %s term code")); break; } } else { @@ -827,18 +832,17 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1)); db_sql_query(jcr->db, query.c_str(), NULL, NULL); } - term_msg = _("%s -- no files to %s"); + Mmsg(term_msg, _("%%s -- no files to %%s")); } - bsnprintf(term_code, sizeof(term_code), term_msg, jcr->get_OperationName(), jcr->get_ActionName(0)); + Mmsg(term_code, term_msg.c_str(), jcr->get_OperationName(), jcr->get_ActionName(0)); bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime); bstrftimes(edt, sizeof(edt), jcr->jr.EndTime); RunTime = jcr->jr.EndTime - jcr->jr.StartTime; if (RunTime <= 0) { - kbps = 0; - } else { - kbps = (double)jcr->SDJobBytes / (1000 * RunTime); + RunTime = 1; } + kbps = (double)jcr->SDJobBytes / (1000.0 * (double)RunTime); jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); @@ -915,7 +919,7 @@ void mac_cleanup(JCR *jcr, int TermCode, int writeTermCode) vol_info.c_str(), jcr->SDErrors, sd_term_msg, - term_code); + term_code.c_str()); Dmsg0(100, "Leave migrate_cleanup()\n"); } diff --git a/bacula/src/dird/msgchan.c b/bacula/src/dird/msgchan.c index ee5f636c9d..6ead5516ff 100644 --- a/bacula/src/dird/msgchan.c +++ b/bacula/src/dird/msgchan.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- msgchan.c -- handles the message channel * to the Storage daemon and the File daemon. * @@ -30,7 +29,6 @@ * to authenticate ourself and to pass the JobId. * Create a thread to interact with the Storage daemon * who returns a job status and requests Catalog services, etc. - * */ #include "bacula.h" @@ -56,7 +54,7 @@ static char OK_device[] = "3000 OK use device device=%s\n"; /* Storage Daemon requests */ static char Job_start[] = "3010 Job %127s start\n"; static char Job_end[] = - "3099 Job %127s end JobStatus=%d JobFiles=%d JobBytes=%lld JobErrors=%u\n"; + "3099 Job %127s end JobStatus=%d JobFiles=%d JobBytes=%lld JobErrors=%u ErrMsg=%256s\n"; /* Forward referenced functions */ extern "C" void *msg_thread(void *arg); @@ -422,8 +420,10 @@ extern "C" void *msg_thread(void *arg) int JobStatus; int n; char Job[MAX_NAME_LENGTH]; + char ErrMsg[256]; uint32_t JobFiles, JobErrors; uint64_t JobBytes; + ErrMsg[0] = 0; pthread_detach(pthread_self()); set_jcr_in_tsd(jcr); @@ -442,11 +442,13 @@ extern "C" void *msg_thread(void *arg) continue; } if (sscanf(sd->msg, Job_end, Job, &JobStatus, &JobFiles, - &JobBytes, &JobErrors) == 5) { + &JobBytes, &JobErrors, ErrMsg) == 6) { jcr->SDJobStatus = JobStatus; /* termination status */ jcr->SDJobFiles = JobFiles; jcr->SDJobBytes = JobBytes; jcr->SDErrors = JobErrors; + unbash_spaces(ErrMsg); /* Error message if any */ + pm_strcpy(jcr->StatusErrMsg, ErrMsg); break; } Dmsg1(400, "end loop use=%d\n", jcr->use_count()); @@ -514,7 +516,7 @@ bool send_bootstrap_file(JCR *jcr, BSOCK *sd) if (!jcr->RestoreBootstrap) { return true; } - bs = fopen(jcr->RestoreBootstrap, "rb"); + bs = bfopen(jcr->RestoreBootstrap, "rb"); if (!bs) { berrno be; Jmsg(jcr, M_FATAL, 0, _("Could not open bootstrap file %s: ERR=%s\n"), diff --git a/bacula/src/dird/next_vol.c b/bacula/src/dird/next_vol.c index f895e5c418..40688a85c3 100644 --- a/bacula/src/dird/next_vol.c +++ b/bacula/src/dird/next_vol.c @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2001-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,19 +11,17 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- next_vol -- handles finding the next * volume for append. Split out of catreq.c August MMIII * catalog request from the Storage daemon. - - * Kern Sibbald, March MMI * + * Kern Sibbald, March MMI */ #include "bacula.h" @@ -33,18 +30,38 @@ static int const dbglvl = 50; /* debug level */ /* - * We setup the StorageId if it is + * We setup the StorageId or StorageId group if it is * an autochanger from the Storage and put it in * the media record. * store == NULL => use existing StorageId */ void set_storageid_in_mr(STORE *store, MEDIA_DBR *mr) { - if (!store) { + if (store == NULL) { + /* Just use the plain (single) StorageId */ + mr->sid_group = edit_int64(mr->StorageId, mr->sid); return; } + /* At this point we know store != NULL */ mr->StorageId = store->StorageId; + /* Get to the parent of the autochanger (if any) */ + if (store->changer) { + store = store->changer; + mr->StorageId = store->StorageId; + } + /* Go to the master shared storage head (if any) */ + if (store->shared_storage && store->shared_storage->ac_group) { + store = store->shared_storage; + } + /* If it is an autochanger we should have an ac_group */ + if (store->autochanger && store->ac_group) { + /* Note we keep the StorageId of the local autochanger */ + mr->sid_group = store->ac_group; + } else { + /* Otherwise, we just use the plain (single) StorageId */ + mr->sid_group = edit_int64(mr->StorageId, mr->sid); + } } static void add_volume_to_exclude_list(JCR *jcr, int index, MEDIA_DBR *mr) diff --git a/bacula/src/dird/protos.h b/bacula/src/dird/protos.h index 9c68e0d688..ffac125539 100644 --- a/bacula/src/dird/protos.h +++ b/bacula/src/dird/protos.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -66,12 +65,13 @@ extern void vbackup_cleanup(JCR *jcr, int TermCode); /* bsr.c */ RBSR *new_bsr(); -void free_bsr(RBSR *bsr); -bool complete_bsr(UAContext *ua, RBSR *bsr); +rblist *create_bsr_list(uint32_t JobId, int findex, int findex2); +void free_bsr(rblist *bsr_list); +bool complete_bsr(UAContext *ua, rblist *bsr_list); uint32_t write_bsr_file(UAContext *ua, RESTORE_CTX &rx); void display_bsr_info(UAContext *ua, RESTORE_CTX &rx); -void add_findex(RBSR *bsr, uint32_t JobId, int32_t findex); -void add_findex_all(RBSR *bsr, uint32_t JobId); +void add_findex(rblist *bsr_list, uint32_t JobId, int32_t findex); +void add_findex_all(rblist *bsr_list, uint32_t JobId, const char *fileregex); RBSR_FINDEX *new_findex(); void make_unique_restore_filename(UAContext *ua, POOLMEM **fname); void print_bsr(UAContext *ua, RESTORE_CTX &rx); @@ -81,10 +81,11 @@ void print_bsr(UAContext *ua, RESTORE_CTX &rx); extern void catalog_request(JCR *jcr, BSOCK *bs); extern void catalog_update(JCR *jcr, BSOCK *bs); extern bool despool_attributes_from_file(JCR *jcr, const char *file); +extern void remove_dummy_jobmedia_records(JCR *jcr); /* dird_conf.c */ extern const char *level_to_str(int level); -extern "C" char *job_code_callback_director(JCR *jcr, const char*); +extern "C" char *job_code_callback_director(JCR *jcr, const char*, char *, int); /* expand.c */ int variable_expansion(JCR *jcr, char *inp, POOLMEM **exp); @@ -126,7 +127,8 @@ extern void apply_pool_overrides(JCR *jcr); extern bool apply_wstorage_overrides(JCR *jcr, POOL *original_pool); extern JobId_t run_job(JCR *jcr); extern JobId_t resume_job(JCR *jcr, JOB_DBR *jr); -extern bool cancel_job(UAContext *ua, JCR *jcr, bool cancel=true); +extern bool cancel_job(UAContext *ua, JCR *jcr, int wait, bool cancel=true); +extern int cancel_inactive_job(UAContext *ua); extern void get_job_storage(USTORE *store, JOB *job, RUN *run); extern void init_jcr_job_record(JCR *jcr); extern void update_job_end(JCR *jcr, int TermCode); @@ -142,6 +144,7 @@ extern void free_rstorage(JCR *jcr); extern bool setup_job(JCR *jcr); extern void create_clones(JCR *jcr); extern int create_restore_bootstrap_file(JCR *jcr); +extern int create_restore_bootstrap_file(JCR *jcr, JobId_t jobid, int findex1, int findex2); extern void dird_free_jcr(JCR *jcr); extern void dird_free_jcr_pointers(JCR *jcr); extern void cancel_storage_daemon_job(JCR *jcr); @@ -197,8 +200,10 @@ extern void restore_cleanup(JCR *jcr, int TermCode); bool acl_access_ok(UAContext *ua, int acl, const char *item); bool acl_access_ok(UAContext *ua, int acl, const char *item, int len); bool have_restricted_acl(UAContext *ua, int acl); +bool acl_access_client_ok(UAContext *ua, const char *name, int32_t jobtype); /* ua_cmds.c */ +bool get_uid_gid_from_acl(UAContext *ua, alist **uid, alist **gid, alist **dir); bool do_a_command(UAContext *ua); bool do_a_dot_command(UAContext *ua); int qmessagescmd(UAContext *ua, const char *cmd); @@ -206,6 +211,7 @@ bool open_new_client_db(UAContext *ua); bool open_client_db(UAContext *ua); bool open_db(UAContext *ua); void close_db(UAContext *ua); +int cloud_volumes_cmd(UAContext *ua, const char *cmd, const char *mode); enum e_pool_op { POOL_OP_UPDATE, POOL_OP_CREATE @@ -261,13 +267,13 @@ JOB *select_job_resource(UAContext *ua); JOB *select_enable_disable_job_resource(UAContext *ua, bool enable); JOB *select_restore_job_resource(UAContext *ua); CLIENT *select_enable_disable_client_resource(UAContext *ua, bool enable); -CLIENT *select_client_resource(UAContext *ua); +CLIENT *select_client_resource(UAContext *ua, int32_t jobtype); FILESET *select_fileset_resource(UAContext *ua); SCHED *select_enable_disable_schedule_resource(UAContext *ua, bool enable); int select_pool_and_media_dbr(UAContext *ua, POOL_DBR *pr, MEDIA_DBR *mr); int select_media_dbr(UAContext *ua, MEDIA_DBR *mr); bool select_pool_dbr(UAContext *ua, POOL_DBR *pr, const char *argk="pool"); -bool select_client_dbr(UAContext *ua, CLIENT_DBR *cr); +bool select_client_dbr(UAContext *ua, CLIENT_DBR *cr, int32_t jobtype); void start_prompt(UAContext *ua, const char *msg); void add_prompt(UAContext *ua, const char *prompt, char *unique=NULL); @@ -280,12 +286,12 @@ int get_storage_drive(UAContext *ua, STORE *store); int get_storage_slot(UAContext *ua, STORE *store); int get_media_type(UAContext *ua, char *MediaType, int max_media); bool get_pool_dbr(UAContext *ua, POOL_DBR *pr, const char *argk="pool"); -bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr); +bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr, int32_t jobtype); POOL *get_pool_resource(UAContext *ua); JOB *get_restore_job(UAContext *ua); POOL *select_pool_resource(UAContext *ua); int select_running_jobs(UAContext *ua, alist *jcrs, const char *reason); -CLIENT *get_client_resource(UAContext *ua); +CLIENT *get_client_resource(UAContext *ua, int32_t jobtype); int get_job_dbr(UAContext *ua, JOB_DBR *jr); int find_arg_keyword(UAContext *ua, const char **list); @@ -295,6 +301,12 @@ int do_keyword_prompt(UAContext *ua, const char *msg, const char **list); int confirm_retention(UAContext *ua, utime_t *ret, const char *msg); int confirm_retention_yesno(UAContext *ua, utime_t ret, const char *msg); bool get_level_from_name(JCR *jcr, const char *level_name); +int get_level_code_from_name(const char *level_name); +int scan_storage_cmd(UAContext *ua, const char *cmd, + bool allfrompool, + int *drive, MEDIA_DBR *mr, POOL_DBR *pr, + const char **action, char *storage, int *nb, uint32_t **results); + /* ua_status.c */ void list_dir_status_header(UAContext *ua); @@ -302,6 +314,7 @@ void list_dir_status_header(UAContext *ua); /* ua_tree.c */ bool user_select_files_from_tree(TREE_CTX *tree); int insert_tree_handler(void *ctx, int num_fields, char **row); +bool check_directory_acl(char **last_dir, alist *dir_acl, const char *path); /* ua_prune.c */ int prune_files(UAContext *ua, CLIENT *client, POOL *pool); @@ -328,6 +341,7 @@ void purge_files_from_job_list(UAContext *ua, del_ctx &del); /* ua_run.c */ extern int run_cmd(UAContext *ua, const char *cmd); extern int restart_cmd(UAContext *ua, const char *cmd); +extern bool check_pool(int32_t JobType, int32_t JobLevel, POOL *pool, POOL *nextpool, const char **name); /* verify.c */ extern bool do_verify(JCR *jcr); diff --git a/bacula/src/dird/recycle.c b/bacula/src/dird/recycle.c index a8665f9615..39e45a997d 100644 --- a/bacula/src/dird/recycle.c +++ b/bacula/src/dird/recycle.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,18 +11,16 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- Automatic Recycling of Volumes * Recycles Volumes that have been purged * * Kern Sibbald, May MMII - * */ @@ -39,7 +37,7 @@ bool find_recycled_volume(JCR *jcr, bool InChanger, MEDIA_DBR *mr, set_storageid_in_mr(store, mr); if (db_find_next_volume(jcr, jcr->db, 1, InChanger, mr)) { jcr->MediaId = mr->MediaId; - Dmsg1(20, "Find_next_vol MediaId=%u\n", jcr->MediaId); + Dmsg1(20, "Find_next_vol MediaId=%lu\n", jcr->MediaId); pm_strcpy(jcr->VolumeName, mr->VolumeName); set_storageid_in_mr(store, mr); return true; diff --git a/bacula/src/dird/restore.c b/bacula/src/dird/restore.c index 1fe0d74231..22359e112c 100644 --- a/bacula/src/dird/restore.c +++ b/bacula/src/dird/restore.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -129,7 +129,7 @@ static bool open_bootstrap_file(JCR *jcr, bootstrap_info &info) } strncpy(info.storage, jcr->rstore->name(), MAX_NAME_LENGTH); - bs = fopen(jcr->RestoreBootstrap, "rb"); + bs = bfopen(jcr->RestoreBootstrap, "rb"); if (!bs) { berrno be; Jmsg(jcr, M_FATAL, 0, _("Could not open bootstrap file %s: ERR=%s\n"), @@ -582,6 +582,7 @@ void restore_cleanup(JCR *jcr, int TermCode) const char *term_msg; int msg_type = M_INFO; double kbps; + utime_t RunTime; Dmsg0(20, "In restore_cleanup\n"); update_job_end(jcr, TermCode); @@ -611,7 +612,7 @@ void restore_cleanup(JCR *jcr, int TermCode) } else if (jcr->JobErrors > 0 || jcr->SDErrors > 0) { term_msg = _("Restore OK -- with errors"); - + } else { term_msg = _("Restore OK"); } @@ -646,11 +647,12 @@ void restore_cleanup(JCR *jcr, int TermCode) } bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime); bstrftimes(edt, sizeof(edt), jcr->jr.EndTime); - if (jcr->jr.EndTime - jcr->jr.StartTime > 0) { - kbps = (double)jcr->jr.JobBytes / (1000 * (jcr->jr.EndTime - jcr->jr.StartTime)); - } else { - kbps = 0; + + RunTime = jcr->jr.EndTime - jcr->jr.StartTime; + if (RunTime <= 0) { + RunTime = 1; } + kbps = (double)jcr->jr.JobBytes / (1000.0 * (double)RunTime); if (kbps < 0.05) { kbps = 0; } diff --git a/bacula/src/dird/scheduler.c b/bacula/src/dird/scheduler.c index 5108142a2d..24ff916bf2 100644 --- a/bacula/src/dird/scheduler.c +++ b/bacula/src/dird/scheduler.c @@ -177,11 +177,11 @@ again: jcr = new_jcr(sizeof(JCR), dird_free_jcr); run = next_job->run; /* pick up needed values */ job = next_job->job; - if (job->enabled && (!job->client || job->client->enabled)) { + if (job->is_enabled() && (!job->client || job->client->is_enabled())) { dump_job(next_job, _("Run job")); /* no client and job enabled */ } free(next_job); - if (!job->enabled || (job->client && !job->client->enabled)) { + if (!job->is_enabled() || (job->client && !job->client->is_enabled())) { free_jcr(jcr); goto again; /* ignore this job */ } @@ -307,8 +307,8 @@ static void find_runs() LockRes(); foreach_res(job, R_JOB) { sched = job->schedule; - if (!sched || !job->enabled || (sched && !sched->enabled) || - (job->client && !job->client->enabled)) { + if (!sched || !job->is_enabled() || (sched && !sched->is_enabled()) || + (job->client && !job->client->is_enabled())) { continue; /* no, skip this job */ } Dmsg1(dbglvl, "Got job: %s\n", job->hdr.name); diff --git a/bacula/src/dird/snapshot.c b/bacula/src/dird/snapshot.c index b6dfcabd73..3dc3b6d10b 100644 --- a/bacula/src/dird/snapshot.c +++ b/bacula/src/dird/snapshot.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,12 +11,10 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. - - Written by Eric Bollengier, 2015 */ #include "bacula.h" @@ -102,7 +100,7 @@ static int get_snapshot_record(UAContext *ua, SNAPSHOT_DBR *snapdbr) return 0; } /* Need to check if the client is authorized */ - if (!acl_access_ok(ua, Client_ACL, snapdbr->Client)) { + if (!acl_access_client_ok(ua, snapdbr->Client, JT_BACKUP_RESTORE)) { Dmsg0(10, "Client access denied\n"); return 0; } @@ -180,7 +178,7 @@ int delete_snapshot(UAContext *ua) /* Try to connect for 15 seconds */ ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); ua->jcr->client = NULL; @@ -217,7 +215,7 @@ int list_snapshot(UAContext *ua, alist *snap_list) CLIENT *client; BSOCK *fd; - client = select_client_resource(ua); + client = select_client_resource(ua, JT_BACKUP_RESTORE); if (!client) { return 0; } @@ -227,7 +225,7 @@ int list_snapshot(UAContext *ua, alist *snap_list) /* Try to connect for 15 seconds */ ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); return 0; @@ -320,7 +318,7 @@ int prune_snapshot(UAContext *ua) /* Try to connect for 15 seconds */ ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); free_bsock(ua->jcr->file_bsock); @@ -608,7 +606,7 @@ int select_snapshot_dbr(UAContext *ua, SNAPSHOT_DBR *sr) CLIENT_DBR cr; memset(&cr, 0, sizeof(cr)); /* Get the pool from client= */ - if (!get_client_dbr(ua, &cr)) { + if (!get_client_dbr(ua, &cr, JT_BACKUP_RESTORE)) { goto bail_out; } sr->ClientId = cr.ClientId; diff --git a/bacula/src/dird/ua.h b/bacula/src/dird/ua.h index aed04ce475..6f8bc3718c 100644 --- a/bacula/src/dird/ua.h +++ b/bacula/src/dird/ua.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2001-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Includes specific to the Director User Agent Server * * Kern Sibbald, August MMI - * */ #ifndef __UA_H_ @@ -63,6 +61,8 @@ public: int32_t int32_val; /* positive/negative */ int64_t int64_val; /* big int */ + void *bvfs; /* used in some bvfs queries */ + void signal(int sig) { UA_sock->signal(sig); }; /* The below are in ua_output.c */ @@ -85,6 +85,10 @@ struct TREE_CTX { uint32_t FileCount; /* current count of files */ uint32_t LastCount; /* last count of files */ uint32_t DeltaCount; /* trigger for printing */ + alist *uid_acl; /* UID allowed in the tree */ + alist *gid_acl; /* GID allowed in the tree */ + alist *dir_acl; /* Directories that can be displayed */ + char *last_dir_acl; /* Last directory from the DirectoryACL list */ }; struct NAME_LIST { @@ -116,7 +120,10 @@ struct RESTORE_CTX { char *where; char *RegexWhere; char *replace; - RBSR *bsr; + char *fileregex; + + char *when; + rblist *bsr_list; POOLMEM *fname; /* filename only */ POOLMEM *path; /* path only */ POOLMEM *query; @@ -125,6 +132,7 @@ struct RESTORE_CTX { bool found; bool all; /* mark all as default */ bool hardlinks_in_mem; /* keep hard links in memory */ + bool fdcalled; /* True if we should reuse the FD socket */ NAME_LIST name_list; POOLMEM *component_fname; FILE *component_fd; diff --git a/bacula/src/dird/ua_acl.c b/bacula/src/dird/ua_acl.c index 1b271bb2b0..5396b1ad09 100644 --- a/bacula/src/dird/ua_acl.c +++ b/bacula/src/dird/ua_acl.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,17 +11,15 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Access Control List (ACL) handling * * Kern Sibbald, January MMIV - * */ #include "bacula.h" @@ -35,6 +33,28 @@ bool acl_access_ok(UAContext *ua, int acl, const char *item) return acl_access_ok(ua, acl, item, strlen(item)); } +bool acl_access_client_ok(UAContext *ua, const char *name, int32_t jobtype) +{ + if (acl_access_ok(ua, Client_ACL, name)) { + return true; + } + if (jobtype == JT_BACKUP && acl_access_ok(ua, BackupClient_ACL, name)) { + return true; + } + if (jobtype == JT_RESTORE && acl_access_ok(ua, RestoreClient_ACL, name)) { + return true; + } + /* Some commands such as "status client" are for both Backup and Restore */ + if (jobtype == JT_BACKUP_RESTORE && + (acl_access_ok(ua, RestoreClient_ACL, name) || + acl_access_ok(ua, BackupClient_ACL, name))) + { + return true; + } + return false; +} + + /* This version expects the length of the item which we must check. */ bool acl_access_ok(UAContext *ua, int acl, const char *item, int len) diff --git a/bacula/src/dird/ua_cmds.c b/bacula/src/dird/ua_cmds.c index 4a3fc965b4..0e1277c16b 100644 --- a/bacula/src/dird/ua_cmds.c +++ b/bacula/src/dird/ua_cmds.c @@ -2,7 +2,6 @@ Bacula(R) - The Network Backup Solution Copyright (C) 2000-2017 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -18,11 +17,9 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Commands * * Kern Sibbald, September MM - * */ #include "bacula.h" @@ -74,6 +71,7 @@ static int time_cmd(UAContext *ua, const char *cmd); static int trace_cmd(UAContext *ua, const char *cmd); static int unmount_cmd(UAContext *ua, const char *cmd); static int use_cmd(UAContext *ua, const char *cmd); +static int cloud_cmd(UAContext *ua, const char *cmd); static int var_cmd(UAContext *ua, const char *cmd); static int version_cmd(UAContext *ua, const char *cmd); static int wait_cmd(UAContext *ua, const char *cmd); @@ -101,10 +99,13 @@ static struct cmdstruct commands[] = { /* C { NT_("autodisplay"), autodisplay_cmd,_("Autodisplay console messages"), NT_("on | off"), false}, { NT_("automount"), automount_cmd, _("Automount after label"), NT_("on | off"), false}, { NT_("cancel"), cancel_cmd, _("Cancel a job"), NT_("jobid= | job= | ujobid= | inactive client= storage= | all"), false}, + { NT_("cloud"), cloud_cmd, _("Specific Cloud commands"), + NT_("[storage=] [volume=] [pool=] [allpools] [allfrompool] [mediatype=] [drive=] [slots="), false}, { NT_("delete"), delete_cmd, _("Delete volume, pool or job"), NT_("volume= | pool= | jobid= | snapshot"), true}, - { NT_("disable"), disable_cmd, _("Disable a job, attributes batch process"), NT_("job= | batch"), true}, - { NT_("enable"), enable_cmd, _("Enable a job, attributes batch process"), NT_("job= | batch"), true}, + { NT_("disable"), disable_cmd, _("Disable a job, attributes batch process"), NT_("job= | client= | schedule= | storage= | batch"), true}, + { NT_("enable"), enable_cmd, _("Enable a job, attributes batch process"), NT_("job= | client= | schedule= | storage= | batch"), true}, { NT_("estimate"), estimate_cmd, _("Performs FileSet estimate, listing gives full listing"), NT_("fileset= client= level= accurate= job= listing"), true}, @@ -116,16 +117,18 @@ static struct cmdstruct commands[] = { /* C "\n\tsetbandwidth setdebug setip show sqlquery time trace unmount\n\tumount update use var version wait" "\n\tsnapshot"), false}, - { NT_("label"), label_cmd, _("Label a tape"), NT_("storage= volume= pool= slot= barcodes"), false}, + { NT_("label"), label_cmd, _("Label a tape"), NT_("storage= volume= pool= slot= drive= barcodes"), false}, { NT_("list"), list_cmd, _("List objects from catalog"), - NT_("jobs [client=] [jobid=] [ujobid=] [job=] [joberrors] [jobstatus=] [limit=]|\n" - "\tjobtotals | pools | volume | media | files jobid= | copies jobid= |\n" - "\tjoblog jobid= | pluginrestoreconf jobid= restoreobjectid= | snapshot"), false}, + NT_("jobs [client=] [jobid=] [ujobid=] [job=] [joberrors] [jobstatus=] [level=] [jobtype=] [limit=]|\n" + "\tjobtotals | pools | volume | media | files [type=] jobid= | copies jobid= |\n" + "\tjoblog jobid= | pluginrestoreconf jobid= restoreobjectid= | snapshot\n" + ), false}, { NT_("llist"), llist_cmd, _("Full or long list like list command"), - NT_("jobs [client=] [jobid=] [ujobid=] [job=] [joberrors] [jobstatus=] [limit=]|\n" + NT_("jobs [client=] [jobid=] [ujobid=] [job=] [joberrors] [jobstatus=] [level=] [jobtype=] [order=] [limit=]|\n" "\tjobtotals | pools | volume | media | files jobid= | copies jobid= |\n" - "\tjoblog jobid= | pluginrestoreconf jobid= restoreobjectid= | snapshot"), false}, + "\tjoblog jobid= | pluginrestoreconf jobid= restoreobjectid= | snapshot |\n" + "\tfileindex=\n"), false}, { NT_("messages"), messagescmd, _("Display pending messages"), NT_(""), false}, { NT_("memory"), memory_cmd, _("Print current memory usage"), NT_(""), true}, @@ -140,8 +143,8 @@ static struct cmdstruct commands[] = { /* C { NT_("query"), query_cmd, _("Query catalog"), NT_("[]"), false}, { NT_("restore"), restore_cmd, _("Restore files"), NT_("where= client= storage= bootstrap= " - "restorejob=" - "\n\tcomment= jobid= copies done select all"), false}, + "restorejob= restoreclient=" + "\n\tcomment= jobid= jobuser= jobgroup= copies done select all"), false}, { NT_("relabel"), relabel_cmd, _("Relabel a tape"), NT_("storage= oldvolume=\n\tvolume= pool="), false}, @@ -162,7 +165,7 @@ static struct cmdstruct commands[] = { /* C "when=\n\tcomment= spooldata= jobid="), false}, { NT_("status"), status_cmd, _("Report status"), - NT_("all | dir= | director | client= |\n" + NT_("all | network [bytes=] | dir= | director | client= |\n" "\tstorage= slots |\n" "\tschedule [job=] [days=] [limit=]\n" "\t\t[time=]"), true}, @@ -172,7 +175,7 @@ static struct cmdstruct commands[] = { /* C NT_("level= tags= trace=0/1 options=<0tTc> tags= | client= | dir | storage= | all"), true}, { NT_("setbandwidth"), setbwlimit_cmd, _("Sets bandwidth"), - NT_("limit= client= jobid= job= ujobid="), true}, + NT_("limit= client= jobid= job= ujobid="), true}, { NT_("snapshot"), snapshot_cmd, _("Handle snapshots"), NT_("[client= | job= | jobid=] [delete | list | listclient | prune | sync | update]"), true}, @@ -193,7 +196,7 @@ static struct cmdstruct commands[] = { /* C { NT_("update"), update_cmd, _("Update volume, pool or stats"), NT_("stats\n\tsnapshot\n\tpool=\n\tslots storage= scan" - "\n\tvolume= volstatus= volretention=" + "\n\tvolume= volstatus= volretention= cacheretention=" "\n\t pool= recycle= slot=\n\t inchanger=" "\n\t maxvolbytes= maxvolfiles= maxvoljobs=" "\n\t enabled= recyclepool= actiononpurge=" @@ -220,8 +223,6 @@ bool do_a_command(UAContext *ua) int len; bool ok = false; bool found = false; - BSOCK *user = ua->UA_sock; - Dmsg1(900, "Command: %s\n", ua->argk[0]); if (ua->argc == 0) { @@ -248,10 +249,10 @@ bool do_a_command(UAContext *ua) ua->error_msg(_("Can't use %s command in a runscript"), ua->argk[0]); break; } - if (ua->api) user->signal(BNET_CMD_BEGIN); + if (ua->api) ua->signal(BNET_CMD_BEGIN); ok = (*commands[i].func)(ua, ua->cmd); /* go execute command */ - if (ua->api) user->signal(ok?BNET_CMD_OK:BNET_CMD_FAILED); - found = (user && user->is_stop()) ? false : true; + if (ua->api) ua->signal(ok?BNET_CMD_OK:BNET_CMD_FAILED); + found = ua->UA_sock && ua->UA_sock->is_stop() ? false : true; break; } } @@ -273,6 +274,7 @@ void set_pool_dbr_defaults_in_media_dbr(MEDIA_DBR *mr, POOL_DBR *pr) bstrncpy(mr->VolStatus, NT_("Append"), sizeof(mr->VolStatus)); mr->Recycle = pr->Recycle; mr->VolRetention = pr->VolRetention; + mr->CacheRetention = pr->CacheRetention; mr->VolUseDuration = pr->VolUseDuration; mr->ActionOnPurge = pr->ActionOnPurge; mr->RecyclePoolId = pr->RecyclePoolId; @@ -466,16 +468,25 @@ static int cancel_cmd(UAContext *ua, const char *cmd) bool cancel = strcasecmp(commands[ua->cmd_index].key, "cancel") == 0; alist *jcrs = New(alist(5, not_owned_by_alist)); + /* If the user explicitely ask, we can send the cancel command to + * the FD. + */ + if (find_arg(ua, "inactive") > 0) { + ret = cancel_inactive_job(ua); + goto bail_out; + } + nb = select_running_jobs(ua, jcrs, commands[ua->cmd_index].key); foreach_alist(jcr, jcrs) { /* Execute the cancel command only if we don't have an error */ if (nb != -1) { - ret &= cancel_job(ua, jcr, cancel); + ret &= cancel_job(ua, jcr, 60, cancel); } free_jcr(jcr); } +bail_out: delete jcrs; return ret; } @@ -510,6 +521,7 @@ void set_pooldbr_from_poolres(POOL_DBR *pr, POOL *pool, e_pool_op op) pr->UseCatalog = pool->use_catalog; pr->Recycle = pool->Recycle; pr->VolRetention = pool->VolRetention; + pr->CacheRetention = pool->CacheRetention; pr->VolUseDuration = pool->VolUseDuration; pr->MaxVolJobs = pool->MaxVolJobs; pr->MaxVolFiles = pool->MaxVolFiles; @@ -684,7 +696,7 @@ extern char *configfile; static int setbwlimit_client(UAContext *ua, CLIENT *client, char *Job, int64_t limit) { CLIENT *old_client; - + char ed1[50]; if (!client) { return 1; } @@ -696,7 +708,7 @@ static int setbwlimit_client(UAContext *ua, CLIENT *client, char *Job, int64_t l /* Try to connect for 15 seconds */ ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); goto bail_out; @@ -708,8 +720,8 @@ static int setbwlimit_client(UAContext *ua, CLIENT *client, char *Job, int64_t l } else { /* Note, we add 2000 OK that was sent by FD to us to message */ - ua->info_msg(_("2000 OK Limiting bandwidth to %lldkb/s %s\n"), - limit/1024, *Job?Job:_("on running and future jobs")); + ua->info_msg(_("2000 OK Limiting bandwidth to %sB/s %s\n"), + edit_uint64_with_suffix(limit, ed1), *Job?Job:_("on running and future jobs")); } ua->jcr->file_bsock->signal(BNET_TERMINATE); @@ -727,7 +739,7 @@ static int setbwlimit_cmd(UAContext *ua, const char *cmd) CLIENT *client = NULL; char Job[MAX_NAME_LENGTH]; *Job=0; - int64_t limit = -1; + uint64_t limit = 0; JCR *jcr = NULL; int i; @@ -745,13 +757,18 @@ static int setbwlimit_cmd(UAContext *ua, const char *cmd) i = find_arg_with_value(ua, "limit"); if (i >= 0) { - limit = atoi(ua->argv[i]) * 1024LL; - } - if (limit < 0) { - if (!get_pint(ua, _("Enter new bandwidth limit kb/s: "))) { + if (!speed_to_uint64(ua->argv[i], strlen(ua->argv[i]), &limit)) { + ua->error_msg(_("Invalid value for limit parameter. Expecting speed.\n")); + return 1; + } + } else { + if (!get_cmd(ua, _("Enter new bandwidth limit: "))) { + return 1; + } + if (!speed_to_uint64(ua->cmd, strlen(ua->cmd), &limit)) { + ua->error_msg(_("Invalid value for limit parameter. Expecting speed.\n")); return 1; } - limit = ua->pint32_val * 1024LL; /* kb/s */ } const char *lst[] = { "job", "jobid", "jobname", NULL }; @@ -767,7 +784,7 @@ static int setbwlimit_cmd(UAContext *ua, const char *cmd) } } else { - client = get_client_resource(ua); + client = get_client_resource(ua, JT_BACKUP_RESTORE); if (client) { setbwlimit_client(ua, client, Job, limit); } @@ -783,8 +800,8 @@ static int setbwlimit_cmd(UAContext *ua, const char *cmd) static int setip_cmd(UAContext *ua, const char *cmd) { CLIENT *client; - char buf[1024]; - if (!ua->cons || !acl_access_ok(ua, Client_ACL, ua->cons->name())) { + char addr[1024]; + if (!ua->cons || !acl_access_client_ok(ua, ua->cons->name(), JT_BACKUP_RESTORE)) { ua->error_msg(_("Unauthorized command from this console.\n")); return 1; } @@ -795,15 +812,12 @@ static int setip_cmd(UAContext *ua, const char *cmd) ua->error_msg(_("Client \"%s\" not found.\n"), ua->cons->name()); goto get_out; } - if (client->address) { - free(client->address); - } /* MA Bug 6 remove ifdef */ sockaddr_to_ascii(&(ua->UA_sock->client_addr), - sizeof(ua->UA_sock->client_addr), buf, sizeof(buf)); - client->address = bstrdup(buf); + sizeof(ua->UA_sock->client_addr), addr, sizeof(addr)); + client->setAddress(bstrdup(addr)); ua->send_msg(_("Client \"%s\" address set to %s\n"), - client->name(), client->address); + client->name(), addr); get_out: UnlockRes(); return 1; @@ -851,7 +865,7 @@ static void do_enable_disable_cmd(UAContext *ua, bool setting) ua->error_msg(_("Unauthorized command from this console.\n")); return; } - job->enabled = setting; + job->setEnabled(setting); ua->send_msg(_("Job \"%s\" %sabled\n"), job->name(), setting?"en":"dis"); } @@ -869,11 +883,11 @@ static void do_enable_disable_cmd(UAContext *ua, bool setting) } } if (client) { - if (!acl_access_ok(ua, Client_ACL, client->name())) { + if (!acl_access_client_ok(ua, client->name(), JT_BACKUP_RESTORE)) { ua->error_msg(_("Unauthorized command from this console.\n")); return; } - client->enabled = setting; + client->setEnabled(setting); ua->send_msg(_("Client \"%s\" %sabled\n"), client->name(), setting?"en":"dis"); } @@ -895,7 +909,7 @@ static void do_enable_disable_cmd(UAContext *ua, bool setting) ua->error_msg(_("Unauthorized command from this console.\n")); return; } - sched->enabled = setting; + sched->setEnabled(setting); ua->send_msg(_("Schedule \"%s\" %sabled\n"), sched->name(), setting?"en":"dis"); } @@ -983,7 +997,7 @@ static void do_client_setdebug(UAContext *ua, CLIENT *client, ua->jcr->client = client; /* Try to connect for 15 seconds */ ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); ua->jcr->client = old_client; @@ -1072,7 +1086,7 @@ static void do_all_setdebug(UAContext *ua, int64_t level, while ((client = (CLIENT *)GetNextRes(R_CLIENT, (RES *)client))) { found = 0; for (j=0; jaddress, client->address) == 0 && + if (strcmp(unique_client[j]->address(), client->address()) == 0 && unique_client[j]->FDport == client->FDport) { found = 1; break; @@ -1080,7 +1094,7 @@ static void do_all_setdebug(UAContext *ua, int64_t level, } if (!found) { unique_client[i++] = client; - Dmsg2(140, "Stuffing: %s:%d\n", client->address, client->FDport); + Dmsg2(140, "Stuffing: %s:%d\n", client->address(), client->FDport); } } UnlockRes(); @@ -1182,7 +1196,7 @@ static int setdebug_cmd(UAContext *ua, const char *cmd) return 1; } } - client = select_client_resource(ua); + client = select_client_resource(ua, JT_BACKUP_RESTORE); if (client) { do_client_setdebug(ua, client, level, trace_flag, hangup, blowup, options, tags_str); @@ -1231,7 +1245,7 @@ static int setdebug_cmd(UAContext *ua, const char *cmd) } break; case 2: - client = select_client_resource(ua); + client = select_client_resource(ua, JT_BACKUP_RESTORE); if (client) { do_client_setdebug(ua, client, level, trace_flag, hangup, blowup, options, tags_str); @@ -1297,7 +1311,10 @@ static int estimate_cmd(UAContext *ua, const char *cmd) JCR *jcr = ua->jcr; int accurate=-1; + jcr->setJobType(JT_BACKUP); + jcr->start_time = time(NULL); jcr->setJobLevel(L_FULL); + for (int i=1; iargc; i++) { if (strcasecmp(ua->argk[i], NT_("client")) == 0 || strcasecmp(ua->argk[i], NT_("fd")) == 0) { @@ -1307,7 +1324,7 @@ static int estimate_cmd(UAContext *ua, const char *cmd) ua->error_msg(_("Client \"%s\" not found.\n"), ua->argv[i]); return 1; } - if (!acl_access_ok(ua, Client_ACL, client->name())) { + if (!acl_access_client_ok(ua, client->name(), JT_BACKUP)) { ua->error_msg(_("No authorization for Client \"%s\"\n"), client->name()); return 1; } @@ -1398,6 +1415,7 @@ static int estimate_cmd(UAContext *ua, const char *cmd) return 1; } } + jcr->job = job; if (!client) { client = job->client; } @@ -1417,9 +1435,6 @@ static int estimate_cmd(UAContext *ua, const char *cmd) return 1; } - jcr->job = job; - jcr->setJobType(JT_BACKUP); - jcr->start_time = time(NULL); init_jcr_job_record(jcr); if (!get_or_create_client_record(jcr)) { @@ -1432,7 +1447,7 @@ static int estimate_cmd(UAContext *ua, const char *cmd) get_level_since_time(ua->jcr, since, sizeof(since)); ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - jcr->client->name(), jcr->client->address, jcr->client->FDport); + jcr->client->name(), jcr->client->address(), jcr->client->FDport); if (!connect_to_file_daemon(jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); return 1; @@ -1722,8 +1737,15 @@ static void do_storage_cmd(UAContext *ua, const char *command) pm_strcpy(store.store_source, _("unknown source")); set_wstorage(jcr, &store); drive = get_storage_drive(ua, store.store); - slot = get_storage_slot(ua, store.store); - + /* For the disable/enable command, the slot is not mandatory */ + if (strcasecmp(command, "disable") == 0 || strcasecmp(command, "enable") == 0) { + slot = 0; + } else { + slot = get_storage_slot(ua, store.store); + } + if (slot < 0) { + return; + } /* Users may set a device name directly on the command line */ if ((i = find_arg_with_value(ua, "device")) > 0) { POOLMEM *errmsg = get_pool_memory(PM_NAME); @@ -1749,7 +1771,7 @@ static void do_storage_cmd(UAContext *ua, const char *command) } sd = jcr->store_bsock; bash_spaces(dev_name); - sd->fsend("%s %s drive=%d slot=%d", command, dev_name, drive, slot); + sd->fsend("%s %s drive=%d slot=%d\n", command, dev_name, drive, slot); while (sd->recv() >= 0) { ua->send_msg("%s", sd->msg); } @@ -1786,6 +1808,301 @@ static int release_cmd(UAContext *ua, const char *cmd) return 1; } +/* + * cloud functions, like to upload cached parts to cloud. + */ +int cloud_volumes_cmd(UAContext *ua, const char *cmd, const char *mode) +{ + int drive = -1; + int nb = 0; + uint32_t *results = NULL; + MEDIA_DBR mr; + POOL_DBR pr; + BSOCK *sd = NULL; + char storage[MAX_NAME_LENGTH]; + const char *action = mode; + memset(&pr, 0, sizeof(pr)); + + /* + * Look for all volumes that are enabled and + * have more the 200 bytes. + */ + mr.Enabled = 1; + mr.Recycle = -1; /* All Recycle status */ + if (strcmp("prunecache", mode) == 0) { + mr.CacheRetention = 1; + action = "truncate cache"; + } + + if (!scan_storage_cmd(ua, cmd, false, /* fromallpool*/ + &drive, &mr, &pr, NULL, storage, + &nb, &results)) + { + goto bail_out; + } + + if ((sd=open_sd_bsock(ua)) == NULL) { + Dmsg0(100, "Can't open connection to sd\n"); + goto bail_out; + } + + /* + * Loop over the candidate Volumes and upload parts + */ + for (int i=0; i < nb; i++) { + bool ok=false; + mr.clear(); + mr.MediaId = results[i]; + if (!db_get_media_record(ua->jcr, ua->db, &mr)) { + goto bail_out; + } + + /* Protect us from spaces */ + bash_spaces(mr.VolumeName); + bash_spaces(mr.MediaType); + bash_spaces(pr.Name); + bash_spaces(storage); + + sd->fsend("%s Storage=%s Volume=%s PoolName=%s MediaType=%s " + "Slot=%d drive=%d CacheRetention=%lld\n", + action, storage, mr.VolumeName, pr.Name, mr.MediaType, + mr.Slot, drive, mr.CacheRetention); + + unbash_spaces(mr.VolumeName); + unbash_spaces(mr.MediaType); + unbash_spaces(pr.Name); + unbash_spaces(storage); + + /* Check for valid response */ + while (bget_dirmsg(sd) >= 0) { + if (strncmp(sd->msg, "3000 OK truncate cache", 22) == 0) { + ua->send_msg("%s", sd->msg); + ok = true; + + } else if (strncmp(sd->msg, "3000 OK", 7) == 0) { + ua->send_msg(_("The volume \"%s\" has been uploaded\n"), mr.VolumeName); + ok = true; + + + } else if (strncmp(sd->msg, "39", 2) == 0) { + ua->warning_msg("%s", sd->msg); + + } else { + ua->send_msg("%s", sd->msg); + } + } + if (!ok) { + ua->warning_msg(_("Unable to %s for volume \"%s\"\n"), action, mr.VolumeName); + } + } + +bail_out: + close_db(ua); + close_sd_bsock(ua); + ua->jcr->wstore = NULL; + if (results) { + free(results); + } + + return 1; +} + +/* List volumes in the cloud */ +/* TODO: Update the code for .api 2 and llist */ +static int cloud_list_cmd(UAContext *ua, const char *cmd) +{ + int drive = -1; + int64_t size, mtime; + STORE *store = NULL; + MEDIA_DBR mr; + POOL_DBR pr; + BSOCK *sd = NULL; + char storage[MAX_NAME_LENGTH]; + char ed1[50], ed2[50]; + bool first=true; + uint32_t maxpart=0, part; + uint64_t maxpart_size=0; + memset(&pr, 0, sizeof(pr)); + memset(&mr, 0, sizeof(mr)); + + /* Look at arguments */ + for (int i=1; iargc; i++) { + if (strcasecmp(ua->argk[i], NT_("volume")) == 0 + && is_name_valid(ua->argv[i], NULL)) { + bstrncpy(mr.VolumeName, ua->argv[i], sizeof(mr.VolumeName)); + + } else if (strcasecmp(ua->argk[i], NT_("drive")) == 0 && ua->argv[i]) { + drive = atoi(ua->argv[i]); + } + } + + if (!open_client_db(ua)) { + goto bail_out; + } + + /* Choose storage */ + ua->jcr->wstore = store = get_storage_resource(ua, false); + if (!store) { + goto bail_out; + } + bstrncpy(storage, store->dev_name(), sizeof(storage)); + bstrncpy(mr.MediaType, store->media_type, sizeof(mr.MediaType)); + + if ((sd=open_sd_bsock(ua)) == NULL) { + Dmsg0(100, "Can't open connection to SD\n"); + goto bail_out; + } + + /* Protect us from spaces */ + bash_spaces(mr.MediaType); + bash_spaces(storage); + bash_spaces(mr.VolumeName); + + sd->fsend("cloudlist Storage=%s Volume=%s MediaType=%s Slot=%d drive=%d\n", + storage, mr.VolumeName, mr.MediaType, mr.Slot, drive); + + if (mr.VolumeName[0]) { /* Want to list parts */ + const char *output_hformat="| %8d | %12sB | %20s |\n"; + uint64_t volsize=0; + /* Check for valid response */ + while (sd->recv() >= 0) { + if (sscanf(sd->msg, "part=%d size=%lld mtime=%lld", &part, &size, &mtime) != 3) { + if (sd->msg[0] == '3') { + ua->send_msg("%s", sd->msg); + } + continue; + } + /* Print information */ + if (first) { + ua->send_msg(_("+----------+---------------+----------------------+\n")); + ua->send_msg(_("| Part | Size | MTime |\n")); + ua->send_msg(_("+----------+---------------+----------------------+\n")); + first=false; + } + if (part > maxpart) { + maxpart = part; + maxpart_size = size; + } + volsize += size; + ua->send_msg(output_hformat, part, edit_uint64_with_suffix(size, ed1), bstrftimes(ed2, sizeof(ed2), mtime)); + } + if (!first) { + ua->send_msg(_("+----------+---------------+----------------------+\n")); + } + /* TODO: See if we fix the catalog record directly */ + if (db_get_media_record(ua->jcr, ua->db, &mr)) { + POOL_MEM errmsg, tmpmsg; + if (mr.LastPartBytes != maxpart_size) { + Mmsg(tmpmsg, "Error on volume \"%s\". Catalog LastPartBytes mismatch %lld != %lld\n", + mr.VolumeName, mr.LastPartBytes, maxpart_size); + pm_strcpy(errmsg, tmpmsg.c_str()); + } + if (mr.VolCloudParts != maxpart) { + Mmsg(tmpmsg, "Error on volume \"%s\". Catalog VolCloudParts mismatch %ld != %ld\n", + mr.VolumeName, mr.VolCloudParts, maxpart); + pm_strcpy(errmsg, tmpmsg.c_str()); + } + if (strlen(errmsg.c_str()) > 0) { + ua->error_msg("\n%s", errmsg.c_str()); + } + } + } else { /* TODO: Get the last part if possible? */ + const char *output_hformat="| %18s | %9s | %20s | %20s | %12sB |\n"; + + /* Check for valid response */ + while (sd->recv() >= 0) { + if (sscanf(sd->msg, "volume=%127s", mr.VolumeName) != 1) { + if (sd->msg[0] == '3') { + ua->send_msg("%s", sd->msg); + } + continue; + } + unbash_spaces(mr.VolumeName); + + mr.MediaId = 0; + + if (mr.VolumeName[0] && db_get_media_record(ua->jcr, ua->db, &mr)) { + memset(&pr, 0, sizeof(POOL_DBR)); + pr.PoolId = mr.PoolId; + if (!db_get_pool_record(ua->jcr, ua->db, &pr)) { + strcpy(pr.Name, "?"); + } + + if (first) { + ua->send_msg(_("+--------------------+-----------+----------------------+----------------------+---------------+\n")); + ua->send_msg(_("| Volume Name | Status | Media Type | Pool | VolBytes |\n")); + ua->send_msg(_("+--------------------+-----------+----------------------+----------------------+---------------+\n")); + first=false; + } + /* Print information */ + ua->send_msg(output_hformat, mr.VolumeName, mr.VolStatus, mr.MediaType, pr.Name, + edit_uint64_with_suffix(mr.VolBytes, ed1)); + } + } + if (!first) { + ua->send_msg(_("+--------------------+-----------+----------------------+----------------------+---------------+\n")); + } + } + +bail_out: + close_db(ua); + close_sd_bsock(ua); + ua->jcr->wstore = NULL; + return 1; +} + +/* Ask client to create/prune/delete a snapshot via the command line */ +static int cloud_cmd(UAContext *ua, const char *cmd) +{ + for (int i=0; iargc; i++) { + if (strcasecmp(ua->argk[i], NT_("upload")) == 0) { + return cloud_volumes_cmd(ua, cmd, "upload"); + + } else if (strcasecmp(ua->argk[i], NT_("list")) == 0) { + return cloud_list_cmd(ua, cmd); + + } else if (strcasecmp(ua->argk[i], NT_("truncate")) == 0) { + return cloud_volumes_cmd(ua, cmd, "truncate cache"); + + } else if (strcasecmp(ua->argk[i], NT_("status")) == 0) { + + } else if (strcasecmp(ua->argk[i], NT_("prune")) == 0) { + return cloud_volumes_cmd(ua, cmd, "prunecache"); + + } else { + continue; + } + } + + for ( ;; ) { + + start_prompt(ua, _("Cloud choice: \n")); + add_prompt(ua, _("List Cloud Volumes in the Cloud")); + add_prompt(ua, _("Upload a Volume to the Cloud")); + add_prompt(ua, _("Prune the Cloud Cache")); + add_prompt(ua, _("Truncate a Volume Cache")); + add_prompt(ua, _("Done")); + + switch(do_prompt(ua, "", _("Select action to perform on Cloud"), NULL, 0)) { + case 0: /* list cloud */ + cloud_list_cmd(ua, cmd); + break; + case 1: /* upload */ + cloud_volumes_cmd(ua, cmd, "upload"); + break; + case 2: /* Prune cache */ + cloud_volumes_cmd(ua, cmd, "prunecache"); + break; + case 3: /* Truncate cache */ + cloud_volumes_cmd(ua, cmd, "truncate cache"); + break; + default: + ua->info_msg(_("Selection terminated.\n")); + return 1; + } + } + return 1; +} /* * Switch databases @@ -1910,7 +2227,9 @@ int wait_cmd(UAContext *ua, const char *cmd) for (bool waiting=false; !waiting; ) { foreach_jcr(jcr) { if (!jcr->is_internal_job() && - (jcr->JobStatus == JS_WaitMedia || jcr->JobStatus == JS_WaitMount)) { + (jcr->JobStatus == JS_WaitMedia || jcr->JobStatus == JS_WaitMount || + jcr->SDJobStatus == JS_WaitMedia || jcr->SDJobStatus == JS_WaitMount)) + { waiting = true; break; } @@ -2132,7 +2451,7 @@ bool open_client_db(UAContext *ua) /* Try for client keyword */ i = find_arg_with_value(ua, NT_("client")); if (i >= 0) { - if (!acl_access_ok(ua, Client_ACL, ua->argv[i])) { + if (!acl_access_client_ok(ua, ua->argv[i], JT_BACKUP_RESTORE)) { ua->error_msg(_("No authorization for Client \"%s\"\n"), ua->argv[i]); return false; } @@ -2184,6 +2503,11 @@ bool open_db(UAContext *ua) { bool mult_db_conn; + /* With a restricted console, we can't share a SQL connection */ + if (ua->cons) { + ua->force_mult_db_connections = true; + } + /* The force_mult_db_connections is telling us if we modify the * private or the shared link */ @@ -2242,7 +2566,24 @@ bool open_db(UAContext *ua) } else { ua->shared_db = ua->db; } + /* With a restricted console, the DB backend should know restrictions about + * Pool, Job, etc... + */ + if (ua->cons) { + ua->db->set_acl(ua->jcr, DB_ACL_JOB, ua->cons->ACL_lists[Job_ACL]); + ua->db->set_acl(ua->jcr, DB_ACL_CLIENT, ua->cons->ACL_lists[Client_ACL]); + ua->db->set_acl(ua->jcr, DB_ACL_POOL, ua->cons->ACL_lists[Pool_ACL]); + ua->db->set_acl(ua->jcr, DB_ACL_FILESET, ua->cons->ACL_lists[FileSet_ACL]); + /* For RestoreClient and BackupClient, we take also in account the Client list */ + ua->db->set_acl(ua->jcr, DB_ACL_RCLIENT, + ua->cons->ACL_lists[Client_ACL], + ua->cons->ACL_lists[RestoreClient_ACL]); + + ua->db->set_acl(ua->jcr, DB_ACL_BCLIENT, + ua->cons->ACL_lists[Client_ACL], + ua->cons->ACL_lists[BackupClient_ACL]); + } if (!ua->api) { ua->send_msg(_("Using Catalog \"%s\"\n"), ua->catalog->name()); } diff --git a/bacula/src/dird/ua_dotcmds.c b/bacula/src/dird/ua_dotcmds.c index 5fbe13f945..450d25fd3f 100644 --- a/bacula/src/dird/ua_dotcmds.c +++ b/bacula/src/dird/ua_dotcmds.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Commands * These are "dot" commands, i.e. commands preceded * by a period. These commands are meant to be used @@ -25,7 +24,6 @@ * returned results are (supposed to be) predictable. * * Kern Sibbald, April MMII - * */ #include "bacula.h" @@ -47,6 +45,7 @@ extern bool dot_status_cmd(UAContext *ua, const char *cmd); /* Forward referenced functions */ static bool admin_cmds(UAContext *ua, const char *cmd); static bool jobscmd(UAContext *ua, const char *cmd); +static bool dotestimatecmd(UAContext *ua, const char *cmd); static bool filesetscmd(UAContext *ua, const char *cmd); static bool clientscmd(UAContext *ua, const char *cmd); static bool msgscmd(UAContext *ua, const char *cmd); @@ -66,6 +65,7 @@ static bool mediacmd(UAContext *ua, const char *cmd); static bool aopcmd(UAContext *ua, const char *cmd); static bool catalogscmd(UAContext *ua, const char *cmd); +static bool dot_ls_cmd(UAContext *ua, const char *cmd); static bool dot_bvfs_lsdirs(UAContext *ua, const char *cmd); static bool dot_bvfs_lsfiles(UAContext *ua, const char *cmd); static bool dot_bvfs_update(UAContext *ua, const char *cmd); @@ -78,6 +78,9 @@ static bool dot_bvfs_decode_lstat(UAContext *ua, const char *cmd); static bool dot_bvfs_update_fv(UAContext *ua, const char *cmd); static bool dot_bvfs_get_volumes(UAContext *ua, const char *cmd); static bool dot_bvfs_get_jobs(UAContext *ua, const char *cmd); +static bool dot_bvfs_get_bootstrap(UAContext *ua, const char *cmd); +static bool dot_bvfs_get_delta(UAContext *ua, const char *cmd); +static void bvfs_get_filter(UAContext *ua, POOL_MEM &where, char *limit, int len); static bool putfile_cmd(UAContext *ua, const char *cmd); static bool api_cmd(UAContext *ua, const char *cmd); @@ -99,6 +102,7 @@ static struct cmdstruct commands[] = { /* help */ /* can be used in runscript * { NT_(".filesets"), filesetscmd, NULL, false}, { NT_(".help"), dot_help_cmd, NULL, false}, { NT_(".jobs"), jobscmd, NULL, true}, + { NT_(".estimate"), dotestimatecmd, NULL, false}, { NT_(".levels"), levelscmd, NULL, false}, { NT_(".messages"), getmsgscmd, NULL, false}, { NT_(".msgs"), msgscmd, NULL, false}, @@ -120,12 +124,15 @@ static struct cmdstruct commands[] = { /* help */ /* can be used in runscript * { NT_(".bvfs_update"), dot_bvfs_update, NULL, true}, { NT_(".bvfs_get_jobids"), dot_bvfs_get_jobids, NULL, true}, { NT_(".bvfs_get_jobs"), dot_bvfs_get_jobs, NULL, true}, + { NT_(".bvfs_get_bootstrap"), dot_bvfs_get_bootstrap,NULL, true}, { NT_(".bvfs_versions"), dot_bvfs_versions, NULL, true}, + { NT_(".bvfs_get_delta"), dot_bvfs_get_delta, NULL, true}, { NT_(".bvfs_restore"), dot_bvfs_restore, NULL, true}, { NT_(".bvfs_cleanup"), dot_bvfs_cleanup, NULL, true}, { NT_(".bvfs_decode_lstat"),dot_bvfs_decode_lstat,NULL, true}, { NT_(".bvfs_clear_cache"),dot_bvfs_clear_cache,NULL, false}, { NT_(".bvfs_update_fv"),dot_bvfs_update_fv, NULL, true}, + { NT_(".ls"), dot_ls_cmd, NULL, false}, { NT_(".types"), typescmd, NULL, false}, { NT_(".tags"), tagscmd, NULL, false} }; @@ -140,17 +147,16 @@ bool do_a_dot_command(UAContext *ua) int len; bool ok = false; bool found = false; - BSOCK *user = ua->UA_sock; - Dmsg1(1400, "Dot command: %s\n", user?user->msg:""); - if (ua->argc == 0 || !user) { + Dmsg1(1400, "Dot command: %s\n", ua->UA_sock?ua->UA_sock->msg:""); + if (ua->argc == 0 || !ua->UA_sock) { return false; } len = strlen(ua->argk[0]); if (len == 1) { - if (ua->api) user->signal(BNET_CMD_BEGIN); - if (ua->api) user->signal(BNET_CMD_OK); + if (ua->api) ua->signal(BNET_CMD_BEGIN); + if (ua->api) ua->signal(BNET_CMD_OK); return true; /* no op */ } for (i=0; igui; /* Check if command permitted, but "quit" is always OK */ if (strcmp(ua->argk[0], NT_(".quit")) != 0 && + strcmp(ua->argk[0], NT_(".api")) != 0 && !acl_access_ok(ua, Command_ACL, ua->argk[0], len)) { + Dmsg1(100, "not allowed %s\n", ua->cmd); break; } Dmsg1(100, "Cmd: %s\n", ua->cmd); ua->gui = true; - if (ua->api) user->signal(BNET_CMD_BEGIN); + if (ua->api) ua->signal(BNET_CMD_BEGIN); ok = (*commands[i].func)(ua, ua->cmd); /* go execute command */ - if (ua->api) user->signal(ok?BNET_CMD_OK:BNET_CMD_FAILED); + if (ua->api) ua->signal(ok?BNET_CMD_OK:BNET_CMD_FAILED); ua->gui = gui; - found = user->is_stop() ? false : true; + found = ua->UA_sock->is_stop() ? false : true; break; } } @@ -183,10 +191,83 @@ bool do_a_dot_command(UAContext *ua) return ok; } +/* + * Send ls to Client + */ +static bool dot_ls_cmd(UAContext *ua, const char *cmd) +{ + CLIENT *client = NULL; + char *path = NULL; + JCR *jcr = ua->jcr; + int i; + + jcr->setJobLevel(L_FULL); + i = find_arg_with_value(ua, NT_("client")); + if (i > 0) { + client = GetClientResWithName(ua->argv[i]); + if (!client) { + ua->error_msg(_("Client \"%s\" not found.\n"), ua->argv[i]); + return false; + } + if (!acl_access_client_ok(ua, client->name(), JT_BACKUP)) { + ua->error_msg(_("No authorization for Client \"%s\"\n"), client->name()); + return false; + } + + } else { + ua->error_msg(_("Client name missing.\n")); + return false; + } + + i = find_arg_with_value(ua, NT_("path")); + if (i > 0) { + path = ua->argv[i]; + + } else { + ua->error_msg(_("path name missing.\n")); + return false; + } + + jcr->client = client; + + jcr->setJobType(JT_BACKUP); + jcr->start_time = time(NULL); + init_jcr_job_record(jcr); // need job + + ua->send_msg(_("Connecting to Client %s at %s:%d\n"), + jcr->client->name(), jcr->client->address(), jcr->client->FDport); + + if (!connect_to_file_daemon(jcr, 1, 15, 0)) { + ua->error_msg(_("Failed to connect to Client.\n")); + return false; + } + + if (!send_ls_fileset(jcr, path)) { + ua->error_msg(_("Failed to send command to Client.\n")); + goto bail_out; + } + + jcr->file_bsock->fsend("estimate listing=%d\n", 1); + while (jcr->file_bsock->recv() >= 0) { + ua->send_msg("%s", jcr->file_bsock->msg); + } + +bail_out: + if (jcr->file_bsock) { + jcr->file_bsock->signal(BNET_TERMINATE); + free_bsock(ua->jcr->file_bsock); + } + return true; +} + static void bvfs_set_acl(UAContext *ua, Bvfs *bvfs) { + if (!ua) { + return; + } + /* If no console resource => default console and all is permitted */ - if (!ua || !ua->cons) { + if (!ua->cons) { return; } bvfs->set_job_acl(ua->cons->ACL_lists[Job_ACL]); @@ -200,6 +281,7 @@ static bool dot_bvfs_decode_lstat(UAContext *ua, const char *cmd) int32_t LinkFI; struct stat sp; POOL_MEM q; + char buf[32]; int pos = find_arg_with_value(ua, "lstat"); if (pos > 0) { @@ -211,11 +293,13 @@ static bool dot_bvfs_decode_lstat(UAContext *ua, const char *cmd) } decode_stat(ua->argv[pos], &sp, sizeof(sp), &LinkFI); - Mmsg(q, "st_nlink=%lld\nst_mode=%lld\nst_uid=%lld\nst_gid=%lld\nst_size=%lld\n" - "st_blocks=%lld\nst_ino=%lld\nst_ctime=%lld\nst_mtime=%lld\nst_mtime=%lld\n" - "st_dev=%lld\nLinkFI=%lld\n", + encode_mode(sp.st_mode, buf); + Mmsg(q, "st_nlink=%lld\nst_mode=%lld\nperm=%s\nst_uid=%lld\nst_gid=%lld\n" + "st_size=%lld\nst_blocks=%lld\nst_ino=%lld\nst_ctime=%lld\n" + "st_mtime=%lld\nst_atime=%lld\nst_dev=%lld\nLinkFI=%lld\n", (int64_t) sp.st_nlink, (int64_t) sp.st_mode, + buf, (int64_t) sp.st_uid, (int64_t) sp.st_gid, (int64_t) sp.st_size, @@ -313,7 +397,6 @@ static int bvfs_result_handler(void *ctx, int fields, char **row) memset(&statp, 0, sizeof(struct stat)); decode_stat(lstat, &statp, sizeof(statp), &LinkFI); - Dmsg1(100, "type=%s\n", row[0]); if (bvfs_is_dir(row)) { char *path = bvfs_basename_dir(row[BVFS_Name]); @@ -334,6 +417,11 @@ static int bvfs_result_handler(void *ctx, int fields, char **row) } else if (bvfs_is_volume_list(row)) { ua->send_msg("%s\t%s\n", row[BVFS_VolName], row[BVFS_VolInchanger]); + + } else if (bvfs_is_delta_list(row)) { + ua->send_msg("%s\t%s\t%s\t%s\t%s\t%s\t%s\n", row[BVFS_PathId], + row[BVFS_FilenameId], fileid, jobid, + lstat, row[BVFS_DeltaSeq], row[BVFS_JobTDate]); } return 0; @@ -387,6 +475,9 @@ static bool bvfs_parse_arg(UAContext *ua, } for (int i=1; iargc; i++) { + if (!ua->argv[i]) { + continue; + } if (strcasecmp(ua->argk[i], NT_("pathid")) == 0) { if (is_a_number(ua->argv[i])) { *pathid = str_to_int64(ua->argv[i]); @@ -477,7 +568,7 @@ static bool dot_bvfs_restore(UAContext *ua, const char *cmd) fileid = dirid = hardlink = empty; if (!bvfs_parse_arg(ua, &pathid, &path, &jobid, &username, - &limit, &offset)) + &limit, &offset) || !path) { ua->error_msg("Can't find jobid, pathid or path argument\n"); return true; /* not enough param */ @@ -501,7 +592,9 @@ static bool dot_bvfs_restore(UAContext *ua, const char *cmd) if ((i = find_arg_with_value(ua, "hardlink")) >= 0) { hardlink = ua->argv[i]; } - + if ((i = find_arg(ua, "nodelta")) >= 0) { + fs.set_compute_delta(false); + } if (fs.compute_restore_list(fileid, dirid, hardlink, path)) { ua->send_msg("OK\n"); } else { @@ -511,6 +604,66 @@ static bool dot_bvfs_restore(UAContext *ua, const char *cmd) return true; } +/* Get a bootstrap for a given bvfs restore session + * .bvfs_get_bootstrap path=b21xxxxxx + * Volume=Vol1 + * Storage=Store1 + * VolAddress=10 + * VolSessionTime=xxx + * VolSessionId=yyyy + */ +static bool dot_bvfs_get_bootstrap(UAContext *ua, const char *cmd) +{ + RESTORE_CTX rx; /* restore context */ + POOLMEM *buf = get_pool_memory(PM_MESSAGE); + int pos; + + new_rx(&rx); + if (!open_new_client_db(ua)) { + ua->error_msg("ERROR: Unable to open database\n"); + goto bail_out; + } + pos = find_arg_with_value(ua, "path"); + if (pos < 0) { + ua->error_msg("ERROR: Unable to get path argument\n"); + goto bail_out; + } + + insert_table_into_findex_list(ua, &rx, ua->argv[pos]); + + if (rx.bsr_list->size() > 0) { + if (!complete_bsr(ua, rx.bsr_list)) { /* find Vol, SessId, SessTime from JobIds */ + ua->error_msg("ERROR: Unable to construct a valid BSR. Cannot continue.\n"); + goto bail_out; + } + if (!(rx.selected_files = write_bsr_file(ua, rx))) { + ua->error_msg("ERROR: No files selected to be restored.\n"); + goto bail_out; + } + FILE *fp = bfopen(ua->jcr->RestoreBootstrap, "r"); + if (!fp) { + ua->error_msg("ERROR: Unable to open bootstrap file\n"); + goto bail_out; + } + while (bfgets(buf, fp)) { + ua->send_msg("%s", buf); + } + fclose(fp); + } else { + ua->error_msg("ERROR: Unable to find files to restore\n"); + goto bail_out; + } + +bail_out: + if (ua->jcr->unlink_bsr) { + unlink(ua->jcr->RestoreBootstrap); + ua->jcr->unlink_bsr = false; + } + free_pool_memory(buf); + free_rx(&rx); + return true; +} + /* * .bvfs_get_volumes [path=/ filename=test jobid=1 | fileid=1] * Vol001 @@ -553,6 +706,7 @@ static bool dot_bvfs_get_volumes(UAContext *ua, const char *cmd) fs.set_username(username); fs.set_handler(bvfs_result_handler, ua); fs.set_limit(limit); + ua->bvfs = &fs; if (filename) { /* TODO */ @@ -560,7 +714,7 @@ static bool dot_bvfs_get_volumes(UAContext *ua, const char *cmd) } else { fs.get_volumes(fileid); } - + ua->bvfs = NULL; return true; } @@ -574,6 +728,7 @@ static bool dot_bvfs_lsfiles(UAContext *ua, const char *cmd) int limit=2000, offset=0; char *path=NULL, *jobid=NULL, *username=NULL; char *pattern=NULL, *filename=NULL; + bool ok; int i; if (!bvfs_parse_arg(ua, &pathid, &path, &jobid, &username, @@ -599,6 +754,8 @@ static bool dot_bvfs_lsfiles(UAContext *ua, const char *cmd) fs.set_jobids(jobid); fs.set_handler(bvfs_result_handler, ua); fs.set_limit(limit); + fs.set_offset(offset); + ua->bvfs = &fs; if (pattern) { fs.set_pattern(pattern); } @@ -606,15 +763,18 @@ static bool dot_bvfs_lsfiles(UAContext *ua, const char *cmd) fs.set_filename(filename); } if (pathid) { - fs.ch_dir(pathid); + ok = fs.ch_dir(pathid); } else { - fs.ch_dir(path); + ok = fs.ch_dir(path); + } + if (!ok) { + goto bail_out; } - - fs.set_offset(offset); fs.ls_files(); +bail_out: + ua->bvfs = NULL; return true; } @@ -630,6 +790,7 @@ static bool dot_bvfs_lsdirs(UAContext *ua, const char *cmd) char *path=NULL, *jobid=NULL, *username=NULL; char *pattern=NULL; int dironly; + bool ok; int i; if (!bvfs_parse_arg(ua, &pathid, &path, &jobid, &username, @@ -655,27 +816,67 @@ static bool dot_bvfs_lsdirs(UAContext *ua, const char *cmd) fs.set_jobids(jobid); fs.set_limit(limit); fs.set_handler(bvfs_result_handler, ua); + fs.set_offset(offset); + ua->bvfs = &fs; if (pattern) { fs.set_pattern(pattern); } if (pathid) { - fs.ch_dir(pathid); + ok = fs.ch_dir(pathid); } else { - fs.ch_dir(path); + ok = fs.ch_dir(path); } - fs.set_offset(offset); + if (!ok) { + goto bail_out; + } fs.ls_special_dirs(); if (dironly < 0) { fs.ls_dirs(); } +bail_out: + ua->bvfs = NULL; return true; } +/* + * .bvfs_get_delta fileid=10 + * + */ +static bool dot_bvfs_get_delta(UAContext *ua, const char *cmd) +{ + bool ret; + FileId_t fileid=0; + int i; + + if ((i = find_arg_with_value(ua, "fileid")) >= 0) { + if (!is_a_number(ua->argv[i])) { + ua->error_msg("Expecting integer for FileId, got %s\n", ua->argv[i]); + return true; + } + fileid = str_to_int64(ua->argv[i]); + + } else { + ua->error_msg("Expecting FileId\n"); + return true; + } + + if (!open_new_client_db(ua)) { + return 1; + } + Bvfs fs(ua->jcr, ua->db); + bvfs_set_acl(ua, &fs); + fs.set_handler(bvfs_result_handler, ua); + ua->bvfs = &fs; + ret = fs.get_delta(fileid); + ua->bvfs = NULL; + return ret; +} + /* * .bvfs_versions fnid=10 pathid=10 client=xxx copies versions * @@ -711,8 +912,11 @@ static bool dot_bvfs_versions(UAContext *ua, const char *cmd) fs.set_see_copies(copies); fs.set_handler(bvfs_result_handler, ua); fs.set_offset(offset); + ua->bvfs = &fs; + fs.get_all_file_versions(pathid, fnid, client); + ua->bvfs = NULL; return true; } @@ -722,8 +926,10 @@ static bool dot_bvfs_versions(UAContext *ua, const char *cmd) * -> returns the jobid of the job * .bvfs_get_jobids jobid=1 jobname * -> returns the jobname - * .bvfs_get_jobids client=xxx + * .bvfs_get_jobids client=xxx [ujobid=yyyy] [jobname=] [fileset=] [start=] [end=] * -> returns all jobid for the client + * .bvfs_get_jobids client=xxx count + * -> returns the number of jobids for the client * .bvfs_get_jobids jobid=1 all * -> returns needed jobids to restore with all filesets a JobId=1 time * .bvfs_get_jobids job=XXXXX @@ -798,7 +1004,10 @@ static bool dot_bvfs_get_jobids(UAContext *ua, const char *cmd) /* Return all backup jobid for a client */ } else if ((pos = find_arg_with_value(ua, "client")) >= 0) { CLIENT *cli; + POOL_MEM where; + char limit[50]; bool ret; + int nbjobs; cli = GetClientResWithName(ua->argv[pos]); if (!cli) { @@ -807,13 +1016,16 @@ static bool dot_bvfs_get_jobids(UAContext *ua, const char *cmd) return true; } db_lock(ua->db); + + bvfs_get_filter(ua, where, limit, sizeof(limit)); + Mmsg(ua->db->cmd, "SELECT JobId " "FROM Job JOIN Client USING (ClientId) " "WHERE Client.Name = '%s' " - "AND Job.Type = 'B' AND Job.JobStatus IN ('T', 'W') " - "ORDER By JobTDate ASC", - cli->name()); + "AND Job.Type = 'B' AND Job.JobStatus IN ('T', 'W') %s " + "ORDER By JobTDate ASC %s", + cli->name(), where.c_str(), limit); ret = db_sql_query(ua->db, ua->db->cmd, db_list_handler, &jobids); db_unlock(ua->db); @@ -822,9 +1034,15 @@ static bool dot_bvfs_get_jobids(UAContext *ua, const char *cmd) cli->name()); } + nbjobs = fs.set_jobids(jobids.list); + /* Apply the ACL filter on JobIds */ - fs.set_jobids(jobids.list); - ua->send_msg("%s\n", fs.get_jobids()); + if (find_arg(ua, "count") >= 0) { + ua->send_msg("%d\n", nbjobs); + + } else { + ua->send_msg("%s\n", fs.get_jobids()); + } return true; } @@ -886,20 +1104,83 @@ static bool dot_bvfs_get_jobids(UAContext *ua, const char *cmd) static int jobs_handler(void *ctx, int num_field, char **row) { UAContext *ua = (UAContext *)ctx; - ua->send_msg("%s %s %s\n", row[0], row[1], row[2]); + ua->send_msg("%s %s %s %s\n", row[0], row[1], row[2], row[3]); return 0; } -/* .bvfs_get_jobs client=xxx [fileset=yyyy] - * 1 yyyyy Backup1_xxx_xxx_xxxx_xxx - * 2 yyyyy Backup1_xxx_xxx_xxxx_xxx +static char *get_argument(UAContext *ua, const char *arg, char *esc, bool convert) +{ + int pos; + if (((pos = find_arg_with_value(ua, arg)) < 0) || + (strlen(ua->argv[pos]) > MAX_NAME_LENGTH)) + { + return NULL; + } + db_escape_string(ua->jcr, ua->db, esc, + ua->argv[pos], strlen(ua->argv[pos])); + if (convert) { + for (int i=0; esc[i] ; i++) { + if (esc[i] == '*') { + esc[i] = '%'; + } + } + } + return esc; +} + +/* The DB should be locked */ +static void bvfs_get_filter(UAContext *ua, POOL_MEM &where, char *limit, int len) +{ + POOL_MEM tmp; + char esc_name[MAX_ESCAPE_NAME_LENGTH]; + + if (get_argument(ua, "jobname", esc_name, true) != NULL) { + Mmsg(where, "AND Job.Job LIKE '%s' ", esc_name); + } + + if (get_argument(ua, "fileset", esc_name, true) != NULL) { + Mmsg(tmp, "AND FileSet.FileSet LIKE '%s' ", esc_name); + pm_strcat(where, tmp.c_str()); + } + + if (get_argument(ua, "jobid", esc_name, false) != NULL) { + Mmsg(tmp, "AND Job.JobId = '%s' ", esc_name); + pm_strcat(where, tmp.c_str()); + } + + if (get_argument(ua, "ujobid", esc_name, false) != NULL) { + Mmsg(tmp, "AND Job.Job = '%s' ", esc_name); + pm_strcat(where, tmp.c_str()); + } + + if (get_argument(ua, "start", esc_name, false) != NULL) { + Mmsg(tmp, "AND Job.StartTime >= '%s' ", esc_name); + pm_strcat(where, tmp.c_str()); + } + + if (get_argument(ua, "end", esc_name, false) != NULL) { + Mmsg(tmp, "AND Job.EndTime <= '%s' ", esc_name); + pm_strcat(where, tmp.c_str()); + } + + *limit = 0; + if (get_argument(ua, "limit", esc_name, false) != NULL) { + if (is_a_number(esc_name)) { + bsnprintf(limit, len, "LIMIT %s ", esc_name); + } + } +} + +/* .bvfs_get_jobs client=xxx [ujobid=yyyy] [jobname=] [fileset=] [start=] [end=] + * 1 yyyyy 1 Backup1_xxx_xxx_xxxx_xxx + * 2 yyyyy 0 Backup1_xxx_xxx_xxxx_xxx */ static bool dot_bvfs_get_jobs(UAContext *ua, const char *cmd) { - int pos, posj; - POOL_MEM tmp; + int pos; + POOL_MEM where; char esc_cli[MAX_ESCAPE_NAME_LENGTH]; - char esc_job[MAX_ESCAPE_NAME_LENGTH]; + char limit[MAX_ESCAPE_NAME_LENGTH]; if (!open_new_client_db(ua)) { return true; } @@ -910,31 +1191,25 @@ static bool dot_bvfs_get_jobs(UAContext *ua, const char *cmd) return true; } - if (!acl_access_ok(ua, Client_ACL, ua->argv[pos])) { - return true; - } - - posj = find_arg_with_value(ua, "ujobid"); - /* Do a little check on the size of the argument */ - if (posj >= 0 && strlen(ua->argv[posj]) > MAX_NAME_LENGTH) { + /* TODO: Do checks on Jobs, FileSet, etc... */ + if (!acl_access_client_ok(ua, ua->argv[pos], JT_BACKUP_RESTORE)) { return true; } db_lock(ua->db); db_escape_string(ua->jcr, ua->db, esc_cli, ua->argv[pos], strlen(ua->argv[pos])); - if (posj >= 0) { - db_escape_string(ua->jcr, ua->db, esc_job, - ua->argv[posj], strlen(ua->argv[pos])); - Mmsg(tmp, "AND Job.Job = '%s'", esc_job); - } + + bvfs_get_filter(ua, where, limit, sizeof(limit)); + Mmsg(ua->db->cmd, - "SELECT JobId, JobTDate, Job " - "FROM Job JOIN Client USING (ClientId) " + "SELECT JobId, JobTDate, HasCache, Job " + "FROM Job JOIN Client USING (ClientId) JOIN FileSet USING (FileSetId) " "WHERE Client.Name = '%s' AND Job.Type = 'B' AND Job.JobStatus IN ('T', 'W') " - "%s " - "ORDER By JobTDate DESC", - esc_cli, tmp.c_str()); + "%s " + "ORDER By JobTDate DESC %s", + esc_cli, where.c_str(), limit); + db_sql_query(ua->db, ua->db->cmd, jobs_handler, ua); db_unlock(ua->db); return true; @@ -997,7 +1272,7 @@ static void do_client_cmd(UAContext *ua, CLIENT *client, const char *cmd) ua->jcr->client = client; /* Try to connect for 15 seconds */ ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->error_msg(_("Failed to connect to Client.\n")); return; @@ -1057,7 +1332,7 @@ static bool admin_cmds(UAContext *ua, const char *cmd) client = (CLIENT *)GetResWithName(R_CLIENT, ua->argv[i]); } if (!client) { - client = select_client_resource(ua); + client = select_client_resource(ua, JT_SYSTEM); } } @@ -1091,7 +1366,7 @@ static bool admin_cmds(UAContext *ua, const char *cmd) store = get_storage_resource(ua, false/*no default*/); break; case 2: - client = select_client_resource(ua); + client = select_client_resource(ua, JT_BACKUP_RESTORE); break; default: break; @@ -1172,7 +1447,7 @@ static bool putfile_cmd(UAContext *ua, const char *cmd) /* the (intptr_t)ua will allow one file per console session */ make_unique_filename(&name, (intptr_t)ua, (char *)key); - fp = fopen(name, "w"); + fp = bfopen(name, "w"); if (!fp) { berrno be; ua->error_msg("Unable to open destination file. ERR=%s\n", @@ -1211,25 +1486,117 @@ bail_out: return true; } +/* .estimate command */ +static bool dotestimatecmd(UAContext *ua, const char *cmd) +{ + JOB *jres; + JOB_DBR jr; + //FILESET_DBR fr; + //CLIENT_DBR cr; + char *job = NULL, level = 0, *fileset = NULL, *client = NULL; + memset(&jr, 0, sizeof(jr)); + + for (int i = 1 ; i < ua->argc ; i++) { + if (!ua->argv[i]) { + ua->error_msg(_("Invalid argument for %s\n"), ua->argk[i]); + return true; + + } else if (strcasecmp(ua->argk[i], "job") == 0) { + job = ua->argv[i]; + + } else if (strcasecmp(ua->argk[i], "level") == 0) { + level = toupper(ua->argv[i][0]); + + } else if (strcasecmp(ua->argk[i], "fileset") == 0) { + fileset = ua->argv[i]; + + } else if (strcasecmp(ua->argk[i], "client") == 0) { + client = ua->argv[i]; + } + } + if (!job) { + ua->error_msg(_("Invalid argument for job\n")); + return true; + } + if (!acl_access_ok(ua, Job_ACL, job) || + (fileset && !acl_access_ok(ua, FileSet_ACL, fileset)) || + (client && !acl_access_client_ok(ua, client, JT_BACKUP))) + { + ua->error_msg(_("Access to specified Job, FileSet or Client not allowed.\n")); + return true; + } + jres = (JOB *) GetResWithName(R_JOB, job); + if (!jres) { + ua->error_msg(_("Invalid argument for job\n")); + return true; + } + if (!open_client_db(ua)) { + ua->error_msg(_("Unable to open the catalog.\n")); + return true; + } + + bstrncpy(jr.Name, jres->hdr.name, sizeof(jr.Name)); + jr.JobLevel = level ? level : jres->JobLevel; + if (fileset) { + /* Get FileSetId */ + } + if (client) { + /* Get ClientId */ + } + db_lock(ua->db); + if (db_get_job_statistics(ua->jcr, ua->db, &jr)) { + db_unlock(ua->db); + OutputWriter o(ua->api_opts); + char *p = o.get_output(OT_START_OBJ, + OT_JOBLEVEL, "level", jr.JobLevel, + OT_INT, "nbjob", jr.CorrNbJob, + OT_INT, "corrbytes", jr.CorrJobBytes, + OT_SIZE, "jobbytes", jr.JobBytes, + OT_INT, "corrfiles", jr.CorrJobFiles, + OT_INT32, "jobfiles", jr.JobFiles, + OT_INT, "duration", (int)0, + OT_STRING, "job", jres->hdr.name, + OT_END_OBJ, + OT_END); + ua->send_msg("%s", p); + } else { + /* We unlock the DB after the errmsg copy */ + pm_strcpy(ua->jcr->errmsg, ua->db->errmsg); + db_unlock(ua->db); + ua->error_msg("Error with .estimate %s\n", ua->jcr->errmsg); + } + return true; +} + + /* * Can use an argument to filter on JobType - * .jobs [type=B] + * .jobs [type=B] or [type=!B] */ static bool jobscmd(UAContext *ua, const char *cmd) { JOB *job; uint32_t type = 0; + bool exclude=false; int pos; if ((pos = find_arg_with_value(ua, "type")) >= 0) { - type = ua->argv[pos][0]; + if (ua->argv[pos][0] == '!') { + exclude = true; + type = ua->argv[pos][1]; + } else { + type = ua->argv[pos][0]; + } } LockRes(); foreach_res(job, R_JOB) { - if (!type || type == job->JobType) { - if (acl_access_ok(ua, Job_ACL, job->name())) { - ua->send_msg("%s\n", job->name()); + if (type) { + if ((exclude && type == job->JobType) || (!exclude && type != job->JobType)) { + continue; } } + if (acl_access_ok(ua, Job_ACL, job->name())) { + ua->send_msg("%s\n", job->name()); + } } UnlockRes(); return true; @@ -1261,16 +1628,102 @@ static bool catalogscmd(UAContext *ua, const char *cmd) return true; } +/* This is not a good idea to lock the entire resource list to send information + * on the network or query the DNS. So, we don't use the foreach_res() command + * with a global lock and we do a copy of the client list in a specific list to + * avoid any problem, I'm pretty sure we can use the res_head directly without + * a global lock, but it needs testing to avoid race conditions. + */ +class TmpClient +{ +public: + char *name; + char *address; + + TmpClient(char *n, char *a): + name(bstrdup(n)), address(bstrdup(a)) + { + }; + ~TmpClient() { + free(name); + free(address); + }; +}; + static bool clientscmd(UAContext *ua, const char *cmd) { + int i; CLIENT *client; + const char *ip=NULL; + bool found=false; + alist *clientlist = NULL; + TmpClient *elt; + + if ((i = find_arg_with_value(ua, "address")) >= 0) { + ip = ua->argv[i]; + clientlist = New(alist(50, not_owned_by_alist)); + } + + /* This is not a good idea to lock the entire resource list + * to send information on the network or query the DNS. So, + * we don't use the foreach_res() command with a global lock here. + */ LockRes(); foreach_res(client, R_CLIENT) { - if (acl_access_ok(ua, Client_ACL, client->name())) { - ua->send_msg("%s\n", client->name()); + if (acl_access_client_ok(ua, client->name(), JT_BACKUP_RESTORE)) { + if (ip) { + elt = new TmpClient(client->name(), client->address()); + clientlist->append(elt); + + } else { + /* do not check for a specific ip, display everything */ + ua->send_msg("%s\n", client->name()); + } } } UnlockRes(); + + if (!ip) { + return true; + } + + foreach_alist(elt, clientlist) { + /* We look for a client that matches the specific ip address */ + dlist *addr_list=NULL; + IPADDR *ipaddr; + char buf[128]; + const char *errstr; + + if (strcmp(elt->address, ip) == 0) { + found = true; + + } else if ((addr_list = bnet_host2ipaddrs(elt->address, 0, &errstr)) == NULL) { + Dmsg2(10, "bnet_host2ipaddrs() for host %s failed: ERR=%s\n", + elt->address, errstr); + + } else { + /* Try to find the ip address from the list, we might have + * other ways to compare ip addresses + */ + foreach_dlist(ipaddr, addr_list) { + if (strcmp(ip, ipaddr->get_address(buf, sizeof(buf))) == 0) { + found = true; + break; + } + } + free_addresses(addr_list); + } + + if (found) { + ua->send_msg("%s\n", elt->name); + break; + } + } + /* Cleanup the temp list */ + foreach_alist(elt, clientlist) { + delete elt; + } + delete clientlist; return true; } @@ -1314,13 +1767,43 @@ static bool schedulescmd(UAContext *ua, const char *cmd) static bool storagecmd(UAContext *ua, const char *cmd) { STORE *store; + POOL_MEM tmp; + bool unique=false; + alist *already_in = NULL; + + /* .storage unique */ + if (find_arg(ua, "unique") > 0) { + unique=true; + already_in = New(alist(10, owned_by_alist)); + } + LockRes(); foreach_res(store, R_STORAGE) { if (acl_access_ok(ua, Storage_ACL, store->name())) { - ua->send_msg("%s\n", store->name()); + char *elt; + bool display=true; + + if (unique) { + Mmsg(tmp, "%s:%d", store->address, store->SDport); + foreach_alist(elt, already_in) { /* TODO: See if we need a hash or an ordered list here */ + if (strcmp(tmp.c_str(), elt) == 0) { + display = false; + break; + } + } + if (display) { + already_in->append(bstrdup(tmp.c_str())); + } + } + if (display) { + ua->send_msg("%s\n", store->name()); + } } } UnlockRes(); + if (already_in) { + delete already_in; + } return true; } @@ -1401,7 +1884,7 @@ static bool backupscmd(UAContext *ua, const char *cmd) strcmp(ua->argk[2], "fileset") != 0) { return true; } - if (!acl_access_ok(ua, Client_ACL, ua->argv[1]) || + if (!acl_access_client_ok(ua, ua->argv[1], JT_BACKUP_RESTORE) || !acl_access_ok(ua, FileSet_ACL, ua->argv[2])) { ua->error_msg(_("Access to specified Client or FileSet not allowed.\n")); return true; @@ -1576,8 +2059,9 @@ static bool defaultscmd(UAContext *ua, const char *cmd) ua->send_msg("level=%s", level_to_str(job->JobLevel)); ua->send_msg("type=%s", job_type_to_str(job->JobType)); ua->send_msg("fileset=%s", job->fileset->name()); - ua->send_msg("enabled=%d", job->enabled); + ua->send_msg("enabled=%d", job->is_enabled()); ua->send_msg("catalog=%s", job->client?job->client->catalog->name():_("*None*")); + ua->send_msg("priority=%d", job->Priority); } } /* Send Pool defaults */ @@ -1616,7 +2100,7 @@ static bool defaultscmd(UAContext *ua, const char *cmd) if (storage) { ua->send_msg("storage=%s", storage->name()); ua->send_msg("address=%s", storage->address); - ua->send_msg("enabled=%d", storage->enabled); + ua->send_msg("enabled=%d", storage->is_enabled()); ua->send_msg("media_type=%s", storage->media_type); ua->send_msg("sdport=%d", storage->SDport); device = (DEVICE *)storage->device->first(); @@ -1630,13 +2114,13 @@ static bool defaultscmd(UAContext *ua, const char *cmd) } /* Send Client defaults */ else if (strcmp(ua->argk[1], "client") == 0) { - if (!acl_access_ok(ua, Client_ACL, ua->argv[1])) { + if (!acl_access_client_ok(ua, ua->argv[1], JT_BACKUP_RESTORE)) { return true; } CLIENT *client = (CLIENT *)GetResWithName(R_CLIENT, ua->argv[1]); if (client) { ua->send_msg("client=%s", client->name()); - ua->send_msg("address=%s", client->address); + ua->send_msg("address=%s", client->address()); ua->send_msg("fdport=%d", client->FDport); ua->send_msg("file_retention=%s", edit_uint64(client->FileRetention, ed1)); ua->send_msg("job_retention=%s", edit_uint64(client->JobRetention, ed1)); diff --git a/bacula/src/dird/ua_label.c b/bacula/src/dird/ua_label.c index e5cc16cb75..45c6c96904 100644 --- a/bacula/src/dird/ua_label.c +++ b/bacula/src/dird/ua_label.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,17 +11,15 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- Tape labeling commands * * Kern Sibbald, April MMIII - * */ #include "bacula.h" @@ -177,7 +175,7 @@ void update_slots(UAContext *ua) if (!store.store) { return; } - pm_strcpy(store.store_source, _("command line")); + pm_strcpy(store.store_source, _("Command input")); set_wstorage(ua->jcr, &store); drive = get_storage_drive(ua, store.store); @@ -355,7 +353,7 @@ static int do_label(UAContext *ua, const char *cmd, int relabel) if (!store.store) { return 1; } - pm_strcpy(store.store_source, _("command line")); + pm_strcpy(store.store_source, _("Command input")); set_wstorage(ua->jcr, &store); drive = get_storage_drive(ua, store.store); @@ -477,7 +475,7 @@ checkName: bash_spaces(dev_name); sd->fsend("mount %s drive=%d slot=%d", dev_name, drive, mr.Slot); unbash_spaces(dev_name); - while (sd->recv() >= 0) { + while (bget_dirmsg(sd) >= 0) { ua->send_msg("%s", sd->msg); /* Here we can get * 3001 OK mount. Device=xxx or @@ -705,51 +703,45 @@ static bool send_label_request(UAContext *ua, MEDIA_DBR *mr, MEDIA_DBR *omr, dev_name, mr->VolumeName, pr->Name, mr->MediaType, mr->Slot, drive); } - while (sd->recv() >= 0) { + while (bget_dirmsg(sd) >= 0) { ua->send_msg("%s", sd->msg); if (sscanf(sd->msg, "3000 OK label. VolBytes=%llu VolABytes=%lld VolType=%d ", &VolBytes, &VolABytes, &VolType) == 3) { ok = true; - } - } - unbash_spaces(mr->VolumeName); - unbash_spaces(mr->MediaType); - unbash_spaces(pr->Name); - mr->LabelDate = time(NULL); - mr->set_label_date = true; - if (ok) { - if (media_record_exists) { /* we update it */ - mr->VolBytes = VolBytes; - mr->VolABytes = VolABytes; - mr->VolType = VolType; - mr->InChanger = mr->Slot > 0; /* if slot give assume in changer */ - set_storageid_in_mr(ua->jcr->wstore, mr); - if (!db_update_media_record(ua->jcr, ua->db, mr)) { - ua->error_msg("%s", db_strerror(ua->db)); - ok = false; - } - } else { /* create the media record */ - set_pool_dbr_defaults_in_media_dbr(mr, pr); - mr->VolBytes = VolBytes; - mr->VolABytes = VolABytes; - mr->VolType = VolType; - mr->InChanger = mr->Slot > 0; /* if slot give assume in changer */ - mr->Enabled = 1; - set_storageid_in_mr(ua->jcr->wstore, mr); - if (db_create_media_record(ua->jcr, ua->db, mr)) { - ua->info_msg(_("Catalog record for Volume \"%s\", Slot %d successfully created.\n"), - mr->VolumeName, mr->Slot); - /* Update number of volumes in pool */ - pr->NumVols++; - if (!db_update_pool_record(ua->jcr, ua->db, pr)) { + if (media_record_exists) { /* we update it */ + mr->VolBytes = VolBytes; + mr->VolABytes = VolABytes; + mr->VolType = VolType; + mr->InChanger = mr->Slot > 0; /* if slot give assume in changer */ + set_storageid_in_mr(ua->jcr->wstore, mr); + if (!db_update_media_record(ua->jcr, ua->db, mr)) { ua->error_msg("%s", db_strerror(ua->db)); + ok = false; + } + } else { /* create the media record */ + set_pool_dbr_defaults_in_media_dbr(mr, pr); + mr->VolBytes = VolBytes; + mr->VolABytes = VolABytes; + mr->VolType = VolType; + mr->InChanger = mr->Slot > 0; /* if slot give assume in changer */ + mr->Enabled = 1; + set_storageid_in_mr(ua->jcr->wstore, mr); + if (db_create_media_record(ua->jcr, ua->db, mr)) { + ua->info_msg(_("Catalog record for Volume \"%s\", Slot %d successfully created.\n"), + mr->VolumeName, mr->Slot); + /* Update number of volumes in pool */ + pr->NumVols++; + if (!db_update_pool_record(ua->jcr, ua->db, pr)) { + ua->error_msg("%s", db_strerror(ua->db)); + } + } else { + ua->error_msg("%s", db_strerror(ua->db)); + ok = false; } - } else { - ua->error_msg("%s", db_strerror(ua->db)); - ok = false; } } - } else { + } + if (!ok) { ua->error_msg(_("Label command failed for Volume %s.\n"), mr->VolumeName); } return ok; @@ -1190,7 +1182,7 @@ void status_slots(UAContext *ua, STORE *store_r) } store.store = store_r; - pm_strcpy(store.store_source, _("command line")); + pm_strcpy(store.store_source, _("Command input")); set_wstorage(ua->jcr, &store); get_storage_drive(ua, store.store); diff --git a/bacula/src/dird/ua_output.c b/bacula/src/dird/ua_output.c index 70f87f4d41..c8e156a395 100644 --- a/bacula/src/dird/ua_output.c +++ b/bacula/src/dird/ua_output.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Output Commands * I.e. messages, listing database, showing resources, ... * @@ -96,7 +95,7 @@ static void show_disabled_jobs(UAContext *ua) if (!acl_access_ok(ua, Job_ACL, job->name())) { continue; } - if (!job->enabled) { + if (!job->is_enabled()) { if (first) { first = false; ua->send_msg(_("Disabled Jobs:\n")); @@ -122,6 +121,9 @@ static struct showstruct reses[] = { {NT_("filesets"), R_FILESET}, {NT_("pools"), R_POOL}, {NT_("messages"), R_MSGS}, +// {NT_("consoles"), R_CONSOLE}, +// {NT_("jobdefs"), R_JOBDEFS}, +// {NT_{"autochangers"), R_AUTOCHANGER}, {NT_("all"), -1}, {NT_("help"), -2}, {NULL, 0} @@ -142,7 +144,8 @@ int show_cmd(UAContext *ua, const char *cmd) int i, j, type, len; int recurse; char *res_name; - RES *res; + RES_HEAD *reshead = NULL; + RES *res = NULL; Dmsg1(20, "show: %s\n", ua->UA_sock->msg); @@ -154,8 +157,10 @@ int show_cmd(UAContext *ua, const char *cmd) goto bail_out; } - type = 0; res = NULL; + reshead = NULL; + type = 0; + res_name = ua->argk[i]; if (!ua->argv[i]) { /* was a name given? */ /* No name, dump all resources of specified type */ @@ -165,13 +170,14 @@ int show_cmd(UAContext *ua, const char *cmd) if (strncasecmp(res_name, reses[j].res_name, len) == 0) { type = reses[j].type; if (type > 0) { - res = res_head[type-r_first]; + reshead = res_head[type-r_first]; } else { - res = NULL; + reshead = NULL; } break; } } + } else { /* Dump a single resource with specified name */ recurse = 0; @@ -194,7 +200,7 @@ int show_cmd(UAContext *ua, const char *cmd) for (j=r_first; j<=r_last; j++) { /* Skip R_DEVICE since it is really not used or updated */ if (j != R_DEVICE) { - dump_resource(j, res_head[j-r_first], bsendmsg, ua); + dump_each_resource(j, bsendmsg, ua); } } break; @@ -215,7 +221,12 @@ int show_cmd(UAContext *ua, const char *cmd) goto bail_out; /* Dump a specific type */ default: - dump_resource(recurse?type:-type, res, bsendmsg, ua); + if (res) { /* keyword and argument, ie: show job=name */ + dump_resource(recurse?type:-type, res, bsendmsg, ua); + + } else if (reshead) { /* keyword only, ie: show job */ + dump_each_resource(-type, bsendmsg, ua); + } break; } } @@ -303,8 +314,8 @@ bail_out: * list jobmedia job=name * list joblog jobid= * list joblog job=name - * list files jobid= - list files saved for job nn - * list files job=name + * list files [type=] jobid= - list files saved for job nn + * list files [type=] job=name * list pools - list pool records * list jobtotals - list totals for all jobs * list media - list media for given pool (deprecated) @@ -313,7 +324,10 @@ bail_out: * list nextvol job=xx - list the next vol to be used by job * list nextvolume job=xx - same as above. * list copies jobid=x,y,z + * list pluginrestoreconf jobid=x,y,z [id=k] * + * Note: keyword "long" is before the first command on the command + * line results in doing a llist (long listing). */ /* Do long or full listing */ @@ -325,13 +339,17 @@ int llist_cmd(UAContext *ua, const char *cmd) /* Do short or summary listing */ int list_cmd(UAContext *ua, const char *cmd) { - return do_list_cmd(ua, cmd, HORZ_LIST); + if (find_arg(ua, "long") > 0) { + return do_list_cmd(ua, cmd, VERT_LIST); /* do a long list */ + } else { + return do_list_cmd(ua, cmd, HORZ_LIST); /* do a short list */ + } } static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) { POOLMEM *VolumeName; - int jobid, n; + int jobid=0, n; int i, j; JOB_DBR jr; POOL_DBR pr; @@ -355,11 +373,11 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) } else if (!ua->argv[j]) { /* skip */ } else if (strcasecmp(ua->argk[j], NT_("order")) == 0) { - if (strcasecmp(ua->argv[j], NT_("desc")) == 0 || - strcasecmp(ua->argv[j], NT_("descending")) == 0) { + if ((strcasecmp(ua->argv[j], NT_("desc")) == 0) || + strcasecmp(ua->argv[j], NT_("descending")) == 0) { jr.order = 1; - } else if (strcasecmp(ua->argv[j], NT_("asc")) == 0 || - strcasecmp(ua->argv[j], NT_("ascending")) == 0) { + } else if ((strcasecmp(ua->argv[j], NT_("asc")) == 0) || + strcasecmp(ua->argv[j], NT_("ascending")) == 0) { jr.order = 0; } else { ua->error_msg(_("Unknown order type %s\n"), ua->argv[j]); @@ -370,13 +388,28 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) } else if (strcasecmp(ua->argk[j], NT_("jobstatus")) == 0) { if (B_ISALPHA(ua->argv[j][0])) { - jr.JobStatus = ua->argv[j][0]; + jr.JobStatus = ua->argv[j][0]; /* TODO: Check if the code is correct */ + } + } else if (strcasecmp(ua->argk[j], NT_("jobtype")) == 0) { + if (B_ISALPHA(ua->argv[j][0])) { + jr.JobType = ua->argv[j][0]; /* TODO: Check if the code is correct */ } + } else if (strcasecmp(ua->argk[j], NT_("level")) == 0) { + if (strlen(ua->argv[j]) > 1) { + jr.JobLevel = get_level_code_from_name(ua->argv[j]); + + } else if (B_ISALPHA(ua->argv[j][0])) { + jr.JobLevel = ua->argv[j][0]; /* TODO: Check if the code is correct */ + } + } else if (strcasecmp(ua->argk[j], NT_("level")) == 0) { + + } else if (strcasecmp(ua->argk[j], NT_("client")) == 0) { if (is_name_valid(ua->argv[j], NULL)) { CLIENT_DBR cr; memset(&cr, 0, sizeof(cr)); - if(get_client_dbr(ua, &cr)) { + /* Both Backup & Restore wants to list jobs for this client */ + if(get_client_dbr(ua, &cr, JT_BACKUP_RESTORE)) { jr.ClientId = cr.ClientId; } } @@ -437,20 +470,29 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) /* List FILES */ } else if (strcasecmp(ua->argk[i], NT_("files")) == 0) { - + int deleted = 0; /* see only backed up files */ for (j=i+1; jargc; j++) { if (strcasecmp(ua->argk[j], NT_("ujobid")) == 0 && ua->argv[j]) { bstrncpy(jr.Job, ua->argv[j], MAX_NAME_LENGTH); jr.JobId = 0; db_get_job_record(ua->jcr, ua->db, &jr); jobid = jr.JobId; + } else if (strcasecmp(ua->argk[j], NT_("jobid")) == 0 && ua->argv[j]) { jobid = str_to_int64(ua->argv[j]); + + } else if (strcasecmp(ua->argk[j], NT_("type")) == 0 && ua->argv[j]) { + if (strcasecmp(ua->argv[j], NT_("deleted")) == 0) { + deleted = 1; + } else if (strcasecmp(ua->argv[j], NT_("all")) == 0) { + deleted = -1; + } + continue; /* Type should be before the jobid... */ } else { continue; } if (jobid > 0) { - db_list_files_for_job(ua->jcr, ua->db, jobid, prtit, ua); + db_list_files_for_job(ua->jcr, ua->db, jobid, deleted, prtit, ua); } } @@ -512,6 +554,84 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) } else if (strcasecmp(ua->argk[i], NT_("clients")) == 0) { db_list_client_records(ua->jcr, ua->db, prtit, ua, llist); + } else if (strcasecmp(ua->argk[i], NT_("pluginrestoreconf")) == 0) { + ROBJECT_DBR rr; + memset(&rr, 0, sizeof(rr)); + rr.FileType = FT_PLUGIN_CONFIG; + + for (j=i+1; jargc; j++) { + if (strcasecmp(ua->argk[j], NT_("ujobid")) == 0 && ua->argv[j]) { + bstrncpy(jr.Job, ua->argv[j], MAX_NAME_LENGTH); + jr.JobId = 0; + + } else if (strcasecmp(ua->argk[j], NT_("jobid")) == 0 && ua->argv[j]) { + + if (acl_access_jobid_ok(ua, ua->argv[j])) { + + if (is_a_number(ua->argv[j])) { + rr.JobId = str_to_uint64(ua->argv[j]); + + } else if (is_a_number_list(ua->argv[j])) { + /* In this case, loop directly to find if all jobids are + * accessible */ + rr.JobIds = ua->argv[j]; + } + + } else { + ua->error_msg(_("Invalid jobid argument\n")); + return 1; + } + + } else if (((strcasecmp(ua->argk[j], NT_("id")) == 0) || + (strcasecmp(ua->argk[j], NT_("restoreobjectid")) == 0)) + && ua->argv[j]) + { + rr.RestoreObjectId = str_to_uint64(ua->argv[j]); + + } else if (strcasecmp(ua->argk[j], NT_("objecttype")) == 0 && ua->argv[j]) { + if (strcasecmp(ua->argv[j], NT_("PLUGIN_CONFIG")) == 0) { + rr.FileType = FT_PLUGIN_CONFIG; + + } else if (strcasecmp(ua->argv[j], NT_("PLUGIN_CONFIG_FILLED")) == 0) { + rr.FileType = FT_PLUGIN_CONFIG_FILLED; + + } else if (strcasecmp(ua->argv[j], NT_("RESTORE_FIRST")) == 0) { + rr.FileType = FT_RESTORE_FIRST; + + } else if (strcasecmp(ua->argv[j], NT_("ALL")) == 0) { + rr.FileType = 0; + + } else { + ua->error_msg(_("Unknown ObjectType %s\n"), ua->argv[j]); + return 1; + } + + } else { + continue; + } + } + + if (!rr.JobId && !rr.JobIds) { + ua->error_msg(_("list pluginrestoreconf requires jobid argument\n")); + return 1; + } + + /* Display the content of the restore object */ + if (rr.RestoreObjectId > 0) { + /* Here, the JobId and the RestoreObjectId are set */ + if (db_get_restoreobject_record(ua->jcr, ua->db, &rr)) { + ua->send_msg("%s\n", NPRTB(rr.object)); + } else { + Dmsg0(200, "Object not found\n"); + } + + } else { + db_list_restore_objects(ua->jcr, ua->db, &rr, prtit, ua, llist); + } + + db_free_restoreobject_record(ua->jcr, &rr); + return 1; + /* List MEDIA or VOLUMES */ } else if (strcasecmp(ua->argk[i], NT_("media")) == 0 || strcasecmp(ua->argk[i], NT_("volume")) == 0 || @@ -610,6 +730,10 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) || strcasecmp(ua->argk[i], NT_("order")) == 0 || strcasecmp(ua->argk[i], NT_("jobstatus")) == 0 || strcasecmp(ua->argk[i], NT_("client")) == 0 + || strcasecmp(ua->argk[i], NT_("type")) == 0 + || strcasecmp(ua->argk[i], NT_("level")) == 0 + || strcasecmp(ua->argk[i], NT_("jobtype")) == 0 + || strcasecmp(ua->argk[i], NT_("long")) == 0 ) { /* Ignore it */ } else if (strcasecmp(ua->argk[i], NT_("snapshot")) == 0 || @@ -709,8 +833,8 @@ RUN *find_next_run(RUN *run, JOB *job, utime_t &runtime, int ndays) bool is_scheduled; sched = job->schedule; - if (!sched || !job->enabled || (sched && !sched->enabled) || - (job->client && !job->client->enabled)) { + if (!sched || !job->is_enabled() || (sched && !sched->is_enabled()) || + (job->client && !job->client->is_enabled())) { return NULL; /* no nothing to report */ } diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index 53fa0b96d1..4cf0cc5b66 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,18 +11,16 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Database prune Command * Applies retention periods * * Kern Sibbald, February MMII - * */ #include "bacula.h" @@ -122,7 +120,8 @@ int prunecmd(UAContext *ua, const char *cmd) switch (kw) { case 0: /* prune files */ - if (!(client = get_client_resource(ua))) { + /* We restrict the client list to ClientAcl, maybe something to change later */ + if (!(client = get_client_resource(ua, JT_SYSTEM))) { return false; } if (find_arg_with_value(ua, "pool") >= 0) { @@ -142,7 +141,8 @@ int prunecmd(UAContext *ua, const char *cmd) return true; case 1: /* prune jobs */ - if (!(client = get_client_resource(ua))) { + /* We restrict the client list to ClientAcl, maybe something to change later */ + if (!(client = get_client_resource(ua, JT_SYSTEM))) { return false; } if (find_arg_with_value(ua, "pool") >= 0) { @@ -632,7 +632,7 @@ static bool prune_expired_volumes(UAContext *ua) POOL_MEM query(PM_MESSAGE); POOL_MEM filter(PM_MESSAGE); alist *lst=NULL; - int i=0; + int nb=0, i=0; char *val; MEDIA_DBR mr; @@ -678,17 +678,17 @@ static bool prune_expired_volumes(UAContext *ua) db_sql_query(ua->db, query.c_str(), db_string_list_handler, &lst); foreach_alist(val, lst) { + nb++; memset(&mr, 0, sizeof(mr)); bstrncpy(mr.VolumeName, val, sizeof(mr.VolumeName)); db_get_media_record(ua->jcr, ua->db, &mr); Mmsg(query, _("Volume \"%s\""), val); - Dmsg1(100, "Do prune %s\n", query.c_str()); if (confirm_retention(ua, &mr.VolRetention, query.c_str())) { - Dmsg1(100, "Call Prune %s\n", query.c_str()); prune_volume(ua, &mr); } } - + ua->send_msg(_("%d expired volume%s found\n"), + nb, nb>1?"s":""); ok = true; bail_out: @@ -722,7 +722,7 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) /* Prune only Volumes with status "Full", or "Used" */ if (strcmp(mr->VolStatus, "Full") == 0 || strcmp(mr->VolStatus, "Used") == 0) { - Dmsg2(100, "get prune list MediaId=%d Volume %s\n", (int)mr->MediaId, mr->VolumeName); + Dmsg2(100, "get prune list MediaId=%lu Volume %s\n", mr->MediaId, mr->VolumeName); count = get_prune_list_for_volume(ua, mr, &del); Dmsg1(100, "Num pruned = %d\n", count); if (count != 0) { @@ -734,7 +734,7 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) ua->info_msg(_("Found no Job associated with the Volume \"%s\" to prune\n"), mr->VolumeName); } - ok = is_volume_purged(ua, mr, false); + ok = is_volume_purged(ua, mr); } db_unlock(ua->db); @@ -766,7 +766,7 @@ int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) now = (utime_t)time(NULL); edit_int64(now-period, ed2); Mmsg(query, sel_JobMedia, ed1, ed2); - Dmsg3(200, "Now=%d period=%d now-period=%s\n", (int)now, (int)period, + Dmsg3(250, "Now=%d period=%d now-period=%s\n", (int)now, (int)period, ed2); Dmsg1(100, "Query=%s\n", query.c_str()); diff --git a/bacula/src/dird/ua_purge.c b/bacula/src/dird/ua_purge.c index 408af578d3..75b268f7f5 100644 --- a/bacula/src/dird/ua_purge.c +++ b/bacula/src/dird/ua_purge.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -110,7 +110,8 @@ int purge_cmd(UAContext *ua, const char *cmd) } return 1; case 2: /* client */ - client = get_client_resource(ua); + /* We restrict the client list to ClientAcl, maybe something to change later */ + client = get_client_resource(ua, JT_SYSTEM); if (client) { purge_files_from_client(ua, client); } @@ -125,7 +126,8 @@ int purge_cmd(UAContext *ua, const char *cmd) case 1: switch(find_arg_keyword(ua, jobs_keywords)) { case 0: /* client */ - client = get_client_resource(ua); + /* We restrict the client list to ClientAcl, maybe something to change later */ + client = get_client_resource(ua, JT_SYSTEM); if (client) { purge_jobs_from_client(ua, client); } @@ -156,13 +158,15 @@ int purge_cmd(UAContext *ua, const char *cmd) } switch (do_keyword_prompt(ua, _("Choose item to purge"), keywords)) { case 0: /* files */ - client = get_client_resource(ua); + /* We restrict the client list to ClientAcl, maybe something to change later */ + client = get_client_resource(ua, JT_SYSTEM); if (client) { purge_files_from_client(ua, client); } break; case 1: /* jobs */ - client = get_client_resource(ua); + /* We restrict the client list to ClientAcl, maybe something to change later */ + client = get_client_resource(ua, JT_SYSTEM); if (client) { purge_jobs_from_client(ua, client); } @@ -392,7 +396,7 @@ void upgrade_copies(UAContext *ua, char *jobs) int dbtype = ua->db->bdb_get_type_index(); db_lock(ua->db); - + Mmsg(query, uap_upgrade_copies_oldest_job[dbtype], JT_JOB_COPY, jobs, jobs); db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); Dmsg1(050, "Upgrade copies Log sql=%s\n", query.c_str()); @@ -431,8 +435,8 @@ void purge_jobs_from_catalog(UAContext *ua, char *jobs) db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); Dmsg1(050, "Delete RestoreObject sql=%s\n", query.c_str()); - /* The JobId of the Snapshot record is no longer usable - * TODO: Migth want to use a copy for the jobid? + /* The JobId of the Snapshot record is no longer usable + * TODO: Migth want to use a copy for the jobid? */ Mmsg(query, "UPDATE Snapshot SET JobId=0 WHERE JobId IN (%s)", jobs); db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); @@ -495,7 +499,7 @@ bool purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr, bool force) purge_jobs_from_catalog(ua, jobids); } - ua->info_msg(_("%d File%s on Volume \"%s\" purged from catalog.\n"), + ua->info_msg(_("%d Job%s on Volume \"%s\" purged from catalog.\n"), lst.count, lst.count<=1?"":"s", mr->VolumeName); purged = is_volume_purged(ua, mr, force); @@ -561,16 +565,15 @@ bail_out: * Called here to send the appropriate commands to the SD * to do truncate on purge. */ -static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr, - char *pool, char *storage, - int drive, BSOCK *sd) +static void truncate_volume(UAContext *ua, MEDIA_DBR *mr, + char *pool, char *storage, + int drive, BSOCK *sd) { bool ok = false; uint64_t VolBytes = 0; uint64_t VolABytes = 0; uint32_t VolType = 0; - /* TODO: Return if not mr->Recyle ? */ if (!mr->Recycle) { return; } @@ -578,9 +581,10 @@ static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr, /* Do it only if action on purge = truncate is set */ if (!(mr->ActionOnPurge & ON_PURGE_TRUNCATE)) { ua->error_msg(_("\nThe option \"Action On Purge = Truncate\" was not defined in the Pool resource.\n" - "Unable to truncate volume \"%s\"\n"), mr->VolumeName); + "Unable to truncate volume \"%s\"\n"), mr->VolumeName); return; } + /* * Send the command to truncate the volume after purge. If this feature * is disabled for the specific device, this will be a no-op. @@ -595,35 +599,40 @@ static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr, /* Do it by relabeling the Volume, which truncates it */ sd->fsend("relabel %s OldName=%s NewName=%s PoolName=%s " "MediaType=%s Slot=%d drive=%d\n", - storage, - mr->VolumeName, mr->VolumeName, - pool, mr->MediaType, mr->Slot, drive); + storage, + mr->VolumeName, mr->VolumeName, + pool, mr->MediaType, mr->Slot, drive); unbash_spaces(mr->VolumeName); unbash_spaces(mr->MediaType); unbash_spaces(pool); unbash_spaces(storage); - /* Send relabel command, and check for valid response */ - while (sd->recv() >= 0) { + /* Check for valid response. With cloud volumes, the upload of the part.1 can + * generate a dir_update_volume_info() message that is handled by bget_dirmsg() + */ + while (bget_dirmsg(sd) >= 0) { ua->send_msg("%s", sd->msg); if (sscanf(sd->msg, "3000 OK label. VolBytes=%llu VolABytes=%lld VolType=%d ", - &VolBytes, &VolABytes, &VolType) == 3) { - ok = true; + &VolBytes, &VolABytes, &VolType) == 3) { + + ok=true; + mr->VolBytes = VolBytes; + mr->VolABytes = VolABytes; + mr->VolType = VolType; + mr->VolFiles = 0; + mr->VolParts = 1; + mr->VolCloudParts = 0; + mr->LastPartBytes = VolBytes; + + set_storageid_in_mr(NULL, mr); + if (!db_update_media_record(ua->jcr, ua->db, mr)) { + ua->error_msg(_("Can't update volume size in the catalog\n")); + } + ua->send_msg(_("The volume \"%s\" has been truncated\n"), mr->VolumeName); } } - - if (ok) { - mr->VolBytes = VolBytes; - mr->VolABytes = VolABytes; - mr->VolType = VolType; - mr->VolFiles = 0; - set_storageid_in_mr(NULL, mr); - if (!db_update_media_record(ua->jcr, ua->db, mr)) { - ua->error_msg(_("Can't update volume size in the catalog\n")); - } - ua->send_msg(_("The volume \"%s\" has been truncated\n"), mr->VolumeName); - } else { + if (!ok) { ua->warning_msg(_("Unable to truncate volume \"%s\"\n"), mr->VolumeName); } } @@ -632,7 +641,12 @@ static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr, * Implement Bacula bconsole command purge action * purge action=truncate pool= volume= storage= mediatype= * or - * truncate pool= volume= storage= mediatype= + * truncate [cache] pool= volume= storage= mediatype= + * + * If the keyword "cache: is present, then we use the truncate + * command rather than relabel so that the driver can decide + * whether or not it wants to truncate. Note: only the + * Cloud driver permits truncating the cache. * * Note, later we might want to rename this action_on_purge_cmd() as * was the original, but only if we add additional actions such as @@ -641,66 +655,20 @@ static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr, */ int truncate_cmd(UAContext *ua, const char *cmd) { - bool allpools = false; int drive = -1; int nb = 0; uint32_t *results = NULL; const char *action = "truncate"; - STORE *store = NULL; - POOL *pool = NULL; MEDIA_DBR mr; POOL_DBR pr; - BSOCK *sd = NULL; + BSOCK *sd; + char storage[MAX_NAME_LENGTH]; - memset(&pr, 0, sizeof(pr)); - - /* Look at arguments */ - for (int i=1; iargc; i++) { - if (strcasecmp(ua->argk[i], NT_("allpools")) == 0) { - allpools = true; - - } else if (strcasecmp(ua->argk[i], NT_("volume")) == 0 - && is_name_valid(ua->argv[i], NULL)) { - bstrncpy(mr.VolumeName, ua->argv[i], sizeof(mr.VolumeName)); - - } else if (strcasecmp(ua->argk[i], NT_("mediatype")) == 0 - && ua->argv[i]) { - bstrncpy(mr.MediaType, ua->argv[i], sizeof(mr.MediaType)); - - } else if (strcasecmp(ua->argk[i], NT_("drive")) == 0 && ua->argv[i]) { - drive = atoi(ua->argv[i]); - - } else if (strcasecmp(ua->argk[i], NT_("action")) == 0 - && is_name_valid(ua->argv[i], NULL)) { - action = ua->argv[i]; - } - } - - /* Choose storage */ - ua->jcr->wstore = store = get_storage_resource(ua, false); - if (!store) { - goto bail_out; - } - - if (!open_db(ua)) { - Dmsg0(100, "Can't open db\n"); - goto bail_out; - } - - if (!allpools) { - /* force pool selection */ - pool = get_pool_resource(ua); - if (!pool) { - Dmsg0(100, "Can't get pool resource\n"); - goto bail_out; - } - bstrncpy(pr.Name, pool->name(), sizeof(pr.Name)); - if (!db_get_pool_record(ua->jcr, ua->db, &pr)) { - Dmsg0(100, "Can't get pool record\n"); - goto bail_out; - } - mr.PoolId = pr.PoolId; + if (find_arg(ua, "cache") > 0) { + return cloud_volumes_cmd(ua, cmd, "truncate cache"); } + + memset(&pr, 0, sizeof(pr)); /* * Look for all Purged volumes that can be recycled, are enabled and @@ -709,15 +677,10 @@ int truncate_cmd(UAContext *ua, const char *cmd) mr.Recycle = 1; mr.Enabled = 1; mr.VolBytes = 200; - set_storageid_in_mr(store, &mr); bstrncpy(mr.VolStatus, "Purged", sizeof(mr.VolStatus)); - if (!db_get_media_ids(ua->jcr, ua->db, &mr, &nb, &results)) { - Dmsg0(100, "No results from db_get_media_ids\n"); - goto bail_out; - } - if (!nb) { - ua->send_msg(_("No Volumes found to perform \"truncate\" command.\n")); + if (!scan_storage_cmd(ua, cmd, true, /* allfrompool */ + &drive, &mr, &pr, &action, storage, &nb, &results)) { goto bail_out; } @@ -733,12 +696,24 @@ int truncate_cmd(UAContext *ua, const char *cmd) mr.clear(); mr.MediaId = results[i]; if (db_get_media_record(ua->jcr, ua->db, &mr)) { - /* TODO: ask for drive and change Pool */ + if (drive < 0) { + STORE *store = (STORE*)GetResWithName(R_STORAGE, storage); + drive = get_storage_drive(ua, store); + } + + /* Must select Pool if not already done */ + if (pr.PoolId == 0) { + pr.PoolId = mr.PoolId; + if (!db_get_pool_record(ua->jcr, ua->db, &pr)) { + return 1; + } + } if (strcasecmp("truncate", action) == 0) { - do_truncate_on_purge(ua, &mr, pr.Name, store->dev_name(), drive, sd); + truncate_volume(ua, &mr, pr.Name, storage, + drive, sd); } } else { - Dmsg1(0, "Can't find MediaId=%lld\n", (uint64_t)mr.MediaId); + Dmsg1(0, "Can't find MediaId=%lu\n", mr.MediaId); } } diff --git a/bacula/src/dird/ua_query.c b/bacula/src/dird/ua_query.c index 561b335fdc..4e14f0575e 100644 --- a/bacula/src/dird/ua_query.c +++ b/bacula/src/dird/ua_query.c @@ -61,7 +61,7 @@ int query_cmd(UAContext *ua, const char *cmd) if (!open_client_db(ua)) { goto bail_out; } - if ((fd=fopen(query_file, "rb")) == NULL) { + if ((fd=bfopen(query_file, "rb")) == NULL) { berrno be; ua->error_msg(_("Could not open %s: ERR=%s\n"), query_file, be.bstrerror()); diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index 6671fb615c..51fe898e41 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2002-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,13 +11,12 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Database restore Command * Creates a bootstrap file for restoring files and * starts the restore job. @@ -61,6 +59,7 @@ static void get_and_display_basejobs(UAContext *ua, RESTORE_CTX *rx); void new_rx(RESTORE_CTX *rx) { + RBSR *bsr = NULL; memset(rx, 0, sizeof(*rx)); rx->path = get_pool_memory(PM_FNAME); rx->path[0] = 0; @@ -80,10 +79,11 @@ void new_rx(RESTORE_CTX *rx) rx->query = get_pool_memory(PM_FNAME); rx->query[0] = 0; - rx->bsr = new_bsr(); + rx->bsr_list = New(rblist(bsr, &bsr->link)); rx->hardlinks_in_mem = true; } + /* * Restore files * @@ -107,6 +107,10 @@ int restore_cmd(UAContext *ua, const char *cmd) } for (i = 0; i < ua->argc ; i++) { + if (strcasecmp(ua->argk[i], "fdcalled") == 0) { + rx.fdcalled = true; + } + if (!ua->argv[i]) { continue; /* skip if no value given */ } @@ -119,6 +123,9 @@ int restore_cmd(UAContext *ua, const char *cmd) } else if (strcasecmp(ua->argk[i], "where") == 0) { rx.where = ua->argv[i]; + } else if (strcasecmp(ua->argk[i], "when") == 0) { + rx.when = ua->argv[i]; + } else if (strcasecmp(ua->argk[i], "replace") == 0) { rx.replace = ua->argv[i]; @@ -139,7 +146,7 @@ int restore_cmd(UAContext *ua, const char *cmd) strcasecmp(ua->argv[i], "false")) { rx.hardlinks_in_mem = false; } - } + } } if (strip_prefix || add_suffix || add_prefix) { @@ -204,9 +211,9 @@ int restore_cmd(UAContext *ua, const char *cmd) break; } - if (rx.bsr->JobId) { + if (rx.bsr_list->size() > 0) { char ed1[50]; - if (!complete_bsr(ua, rx.bsr)) { /* find Vol, SessId, SessTime from JobIds */ + if (!complete_bsr(ua, rx.bsr_list)) { /* find Vol, SessId, SessTime from JobIds */ ua->error_msg(_("Unable to construct a valid BSR. Cannot continue.\n")); goto bail_out; } @@ -279,6 +286,15 @@ int restore_cmd(UAContext *ua, const char *cmd) pm_strcat(ua->cmd, buf); } + if (rx.fdcalled) { + pm_strcat(ua->cmd, " fdcalled=yes"); + } + + if (rx.when) { + Mmsg(buf, " when=\"%s\"", rx.when); + pm_strcat(ua->cmd, buf); + } + if (rx.comment) { Mmsg(buf, " comment=\"%s\"", rx.comment); pm_strcat(ua->cmd, buf); @@ -307,6 +323,9 @@ int restore_cmd(UAContext *ua, const char *cmd) * line. */ /* ***FIXME*** pass jobids on command line */ + if (jcr->JobIds) { + free_pool_memory(jcr->JobIds); + } jcr->JobIds = rx.JobIds; rx.JobIds = NULL; jcr->component_fname = rx.component_fname; @@ -332,6 +351,12 @@ bail_out: bfree(regexp); } + /* Free the plugin config if needed, we don't want to re-use + * this part of the next try + */ + free_plugin_config_items(jcr->plugin_config); + jcr->plugin_config = NULL; + free_rx(&rx); garbage_collect_memory(); /* release unused memory */ return 0; @@ -360,13 +385,17 @@ static void get_and_display_basejobs(UAContext *ua, RESTORE_CTX *rx) void free_rx(RESTORE_CTX *rx) { - free_bsr(rx->bsr); - rx->bsr = NULL; + free_bsr(rx->bsr_list); + rx->bsr_list = NULL; free_and_null_pool_memory(rx->JobIds); free_and_null_pool_memory(rx->BaseJobIds); free_and_null_pool_memory(rx->fname); free_and_null_pool_memory(rx->path); free_and_null_pool_memory(rx->query); + if (rx->fileregex) { + free(rx->fileregex); + rx->fileregex = NULL; + } if (rx->component_fd) { fclose(rx->component_fd); rx->component_fd = NULL; @@ -409,7 +438,8 @@ static int get_client_name(UAContext *ua, RESTORE_CTX *rx) return 1; } memset(&cr, 0, sizeof(cr)); - if (!get_client_dbr(ua, &cr)) { + /* We want the name of the client where the backup was made */ + if (!get_client_dbr(ua, &cr, JT_BACKUP_RESTORE)) { return 0; } bstrncpy(rx->ClientName, cr.Name, sizeof(rx->ClientName)); @@ -504,6 +534,10 @@ static int user_select_jobids_or_files(UAContext *ua, RESTORE_CTX *rx) "comment", /* 21 */ "restorejob", /* 22 */ "replace", /* 23 */ + "xxxxxxxxx", /* 24 */ + "fdcalled", /* 25 */ + "when", /* 26 */ + NULL }; @@ -867,8 +901,8 @@ static int user_select_jobids_or_files(UAContext *ua, RESTORE_CTX *rx) pm_strcat(JobIds, edit_int64(JobId, ed1)); rx->TotalFiles += jr.JobFiles; } - free_pool_memory(rx->JobIds); - rx->JobIds = JobIds; /* Set ACL filtered list */ + pm_strcpy(rx->JobIds, JobIds); /* Set ACL filtered list */ + free_pool_memory(JobIds); if (*rx->JobIds == 0) { ua->warning_msg(_("No Jobs selected.\n")); return 0; @@ -915,7 +949,7 @@ static void insert_one_file_or_dir(UAContext *ua, RESTORE_CTX *rx, char *date, b switch (*p) { case '<': p++; - if ((ffd = fopen(p, "rb")) == NULL) { + if ((ffd = bfopen(p, "rb")) == NULL) { berrno be; ua->error_msg(_("Cannot open file %s: ERR=%s\n"), p, be.bstrerror()); @@ -1075,14 +1109,41 @@ static void split_path_and_filename(UAContext *ua, RESTORE_CTX *rx, char *name) Dmsg2(100, "split path=%s file=%s\n", rx->path, rx->fname); } +static bool can_restore_all_files(UAContext *ua) +{ + alist *lst; + if (ua->cons) { + lst = ua->cons->ACL_lists[Directory_ACL]; + /* ACL not defined, or the first entry is not *all* */ + /* TODO: See if we search for *all* in all the list */ + if (!lst || strcasecmp((char*)lst->get(0), "*all*") != 0) { + return false; + } + if (!lst || strcasecmp((char *)lst->get(0), "*all*") != 0) { + return false; + } + } + return true; +} + static bool ask_for_fileregex(UAContext *ua, RESTORE_CTX *rx) { - if (find_arg(ua, NT_("all")) >= 0) { /* if user enters all on command line */ + bool can_restore=can_restore_all_files(ua); + + if (can_restore && find_arg(ua, NT_("all")) >= 0) { /* if user enters all on command line */ return true; /* select everything */ } + ua->send_msg(_("\n\nFor one or more of the JobIds selected, no files were found,\n" "so file selection is not possible.\n" "Most likely your retention policy pruned the files.\n")); + + if (!can_restore) { + ua->error_msg(_("\nThe current Console has UserId or Directory restrictions. " + "The full restore is not allowed.\n")); + return false; + } + if (get_yesno(ua, _("\nDo you want to restore all the files? (yes|no): "))) { if (ua->pint32_val == 1) return true; @@ -1104,7 +1165,7 @@ static bool ask_for_fileregex(UAContext *ua, RESTORE_CTX *rx) if (*errmsg) { ua->send_msg(_("Regex compile error: %s\n"), errmsg); } else { - rx->bsr->fileregex = bstrdup(ua->cmd); + rx->fileregex = bstrdup(ua->cmd); return true; } } @@ -1127,7 +1188,7 @@ static void add_delta_list_findex(RESTORE_CTX *rx, struct delta_list *lst) if (lst->next) { add_delta_list_findex(rx, lst->next); } - add_findex(rx->bsr, lst->JobId, lst->FileIndex); + add_findex(rx->bsr_list, lst->JobId, lst->FileIndex); } /* @@ -1185,6 +1246,7 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) tree.all = rx->all; tree.hardlinks_in_mem = rx->hardlinks_in_mem; last_JobId = 0; + tree.last_dir_acl = NULL; /* * For display purposes, the same JobId, with different volumes may * appear more than once, however, we only insert it once. @@ -1268,7 +1330,7 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) if (JobId == last_JobId) { continue; /* eliminate duplicate JobIds */ } - add_findex_all(rx->bsr, JobId); + add_findex_all(rx->bsr_list, JobId, rx->fileregex); } } } else { @@ -1298,7 +1360,7 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) Dmsg3(400, "JobId=%lld type=%d FI=%d\n", (uint64_t)node->JobId, node->type, node->FileIndex); /* TODO: optimize bsr insertion when jobid are non sorted */ add_delta_list_findex(rx, node->delta_list); - add_findex(rx->bsr, node->JobId, node->FileIndex); + add_findex(rx->bsr_list, node->JobId, node->FileIndex); /* * Special VSS plugin code to return selected * components. For the moment, it is hard coded @@ -1318,7 +1380,11 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) } } } - + if (tree.uid_acl) { + delete tree.uid_acl; + delete tree.gid_acl; + delete tree.dir_acl; + } free_tree(tree.root); /* free the directory tree */ return OK; } @@ -1351,7 +1417,7 @@ static bool select_backups_before_date(UAContext *ua, RESTORE_CTX *rx, char *dat * Select Client from the Catalog */ memset(&cr, 0, sizeof(cr)); - if (!get_client_dbr(ua, &cr)) { + if (!get_client_dbr(ua, &cr, JT_BACKUP_RESTORE)) { goto bail_out; } bstrncpy(rx->ClientName, cr.Name, sizeof(rx->ClientName)); @@ -1517,7 +1583,7 @@ static int jobid_fileindex_handler(void *ctx, int num_fields, char **row) rx->JobId = JobId; } - add_findex(rx->bsr, rx->JobId, str_to_int64(row[1])); + add_findex(rx->bsr_list, rx->JobId, str_to_int64(row[1])); rx->found = true; rx->selected_files++; return 0; @@ -1614,13 +1680,12 @@ void find_storage_resource(UAContext *ua, RESTORE_CTX &rx, char *Storage, char * ua->info_msg(_("\nWarning Storage is overridden by \"%s\" on the command line.\n"), store->name()); rx.store = store; - bstrncpy(Storage, store->name(), MAX_NAME_LENGTH); /* Return overridden Storage */ + bstrncpy(rx.RestoreMediaType, MediaType, sizeof(rx.RestoreMediaType)); if (strcmp(MediaType, store->media_type) != 0) { - ua->info_msg(_("Warning MediaType overridden by Storage Media Type:\n" - " New Storage MediaType=\"%s\"\n" - " Old Volume MediaType=\"%s\".\n\n"), + ua->info_msg(_("This may not work because of two different MediaTypes:\n" + " Storage MediaType=\"%s\"\n" + " Volume MediaType=\"%s\".\n\n"), store->media_type, MediaType); - bstrncpy(MediaType, store->media_type, MAX_NAME_LENGTH); /* Return overridden MediaType */ } Dmsg2(200, "Set store=%s MediaType=%s\n", rx.store->name(), rx.RestoreMediaType); } diff --git a/bacula/src/dird/ua_run.c b/bacula/src/dird/ua_run.c index e5961c8060..c22b9f4e9c 100644 --- a/bacula/src/dird/ua_run.c +++ b/bacula/src/dird/ua_run.c @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2001-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,17 +11,15 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- Run Command * * Kern Sibbald, December MMI - * */ #include "bacula.h" @@ -60,12 +57,14 @@ public: bool restart; bool done; bool alljobid; + bool fdcalled; int spool_data; bool spool_data_set; int accurate; bool accurate_set; int ignoreduplicatecheck; bool ignoreduplicatecheck_set; + alist *plugin_config; /* List of all plugin_item */ /* Methods */ run_ctx() { memset(this, 0, sizeof(run_ctx)); store = new USTORE; }; @@ -116,11 +115,6 @@ int run_cmd(UAContext *ua, const char *cmd) goto bail_out; } - if (find_arg(ua, NT_("fdcalled")) > 0) { - jcr->file_bsock = dup_bsock(ua->UA_sock); - ua->quit = true; - } - for ( ;; ) { /* * Create JCR to run job. NOTE!!! after this point, free_jcr() @@ -131,9 +125,15 @@ int run_cmd(UAContext *ua, const char *cmd) set_jcr_defaults(jcr, rc.job); jcr->unlink_bsr = ua->jcr->unlink_bsr; /* copy unlink flag from caller */ ua->jcr->unlink_bsr = false; + if (find_arg(ua, NT_("fdcalled")) > 0) { + rc.fdcalled = true; + } } /* Transfer JobIds to new restore Job */ if (ua->jcr->JobIds) { + if (jcr->JobIds) { + free_pool_memory(jcr->JobIds); + } jcr->JobIds = ua->jcr->JobIds; ua->jcr->JobIds = NULL; } @@ -144,6 +144,11 @@ int run_cmd(UAContext *ua, const char *cmd) jcr->component_fd = ua->jcr->component_fd; ua->jcr->component_fd = NULL; } + /* Transfer Plugin Restore Configuration */ + if (ua->jcr->plugin_config) { + jcr->plugin_config = ua->jcr->plugin_config; + ua->jcr->plugin_config = NULL; + } if (!set_run_context_in_jcr(ua, jcr, rc)) { break; /* error get out of while loop */ @@ -222,7 +227,48 @@ bail_out: static JobId_t start_job(UAContext *ua, JCR *jcr, run_ctx &rc) { JobId_t JobId; + char ed1[50]; + + /* Do a final check for the client, the job can change in the previous menu */ + if (jcr->client && jcr->job) { + if (!acl_access_client_ok(ua, jcr->client->name(), jcr->job->JobType)) { + ua->error_msg(_("Job failed. Client \"%s\" not authorized on this console\n"), jcr->client->name()); + free_jcr(jcr); + return 0; + } + } + + /* Do a final check for the where/regexwhere, the job can change in the previous menu */ + if (jcr->getJobType() == JT_RESTORE) { + char *p = jcr->RegexWhere ? jcr->RegexWhere : jcr->job->RegexWhere; + if (p) { + if (!acl_access_ok(ua, Where_ACL, p)) { + ua->error_msg(_("\"RegexWhere\" specification not authorized.\n")); + free_jcr(jcr); + return 0; + } + } else { + p = jcr->where ? jcr->where : jcr->job->RestoreWhere; + if (p) { + if (!acl_access_ok(ua, Where_ACL, p)) { + ua->error_msg(_("\"where\" specification not authorized.\n")); + free_jcr(jcr); + return 0; + } + } + } + } + + /* If we use the fdcalled feature, we keep use the UA socket + * as a FileDaemon socket. We do not use dup_bsock() because + * it doesn't work, when the UA will do a free_bsock() all + * socket childs will be closed as well. + */ + if (rc.fdcalled) { + jcr->file_bsock = ua->UA_sock; + jcr->file_bsock->set_jcr(jcr); + } if (rc.jr.JobStatus == JS_Incomplete) { Dmsg1(100, "Ressuming JobId=%d\n", rc.jr.JobId); JobId = resume_job(jcr, &rc.jr); @@ -234,11 +280,16 @@ static JobId_t start_job(UAContext *ua, JCR *jcr, run_ctx &rc) JobId, jcr->pool->name(), jcr->JobPriority); free_jcr(jcr); /* release jcr */ if (JobId == 0) { - ua->error_msg(_("Job failed.\n")); + ua->error_msg(_("Job %s failed.\n"), edit_int64(rc.jr.JobId, ed1)); + } else { - char ed1[50]; ua->send_msg(_("Job queued. JobId=%s\n"), edit_int64(JobId, ed1)); } + if (rc.fdcalled) { + ua->signal(BNET_FDCALLED); /* After this point, this is a new connection */ + ua->UA_sock = new_bsock(); + ua->quit = true; + } return JobId; } @@ -341,24 +392,19 @@ static bool get_next_pool(UAContext *ua, run_ctx &rc) */ static bool get_client(UAContext *ua, run_ctx &rc) { + bool authorized=false; if (rc.client_name) { rc.client = GetClientResWithName(rc.client_name); if (!rc.client) { if (*rc.client_name != 0) { ua->warning_msg(_("Client \"%s\" not found.\n"), rc.client_name); } - rc.client = select_client_resource(ua); + rc.client = select_client_resource(ua, rc.job->JobType); } } else if (!rc.client) { rc.client = rc.job->client; /* use default */ } - if (!rc.client) { - return false; - } else if (!acl_access_ok(ua, Client_ACL, rc.client->name())) { - ua->error_msg(_("No authorization. Client \"%s\".\n"), - rc.client->name()); - return false; - } + Dmsg1(800, "Using client=%s\n", rc.client->name()); if (rc.restore_client_name) { @@ -367,14 +413,19 @@ static bool get_client(UAContext *ua, run_ctx &rc) if (*rc.restore_client_name != 0) { ua->warning_msg(_("Restore Client \"%s\" not found.\n"), rc.restore_client_name); } - rc.client = select_client_resource(ua); + rc.client = select_client_resource(ua, rc.job->JobType); } } else if (!rc.client) { rc.client = rc.job->client; /* use default */ } + if (!rc.client) { return false; - } else if (!acl_access_ok(ua, Client_ACL, rc.client->name())) { + + } else if (acl_access_client_ok(ua, rc.client->name(), rc.job->JobType)) { + authorized = true; + } + if (!authorized) { ua->error_msg(_("No authorization. Client \"%s\".\n"), rc.client->name()); return false; @@ -419,7 +470,7 @@ static bool get_storage(UAContext *ua, run_ctx &rc) { if (rc.store_name) { rc.store->store = GetStoreResWithName(rc.store_name); - pm_strcpy(rc.store->store_source, _("command line")); + pm_strcpy(rc.store->store_source, _("Command input")); if (!rc.store->store) { if (*rc.store_name != 0) { ua->warning_msg(_("Storage \"%s\" not found.\n"), rc.store_name); @@ -476,7 +527,14 @@ static bool get_jobid_list(UAContext *ua, sellist &sl, run_ctx &rc) return false; } } - jr.limit = 100; /* max 100 records */ + + if ((i=find_arg_with_value(ua, "limit")) >= 0) { + jr.limit = str_to_int64(ua->argv[i]); + + } else { + jr.limit = 100; /* max 100 records */ + } + if (rc.job_name) { bstrncpy(jr.Name, rc.job_name, sizeof(jr.Name)); } else { @@ -646,6 +704,362 @@ int restart_cmd(UAContext *ua, const char *cmd) return 0; /* do not run */ } + +/* + * Plugin restore option part + */ + +/* Free a plugin_config_item */ +void free_plugin_config_item(plugin_config_item *elt) +{ + free(elt->plugin_name); + free_pool_memory(elt->content); + free(elt); +} + +/* Free a list of plugins (do not free the list itself) */ +void free_plugin_config_items(alist *lst) +{ + plugin_config_item *elt; + + if (!lst) { + return; + } + + foreach_alist(elt, lst) { + free_plugin_config_item(elt); + } +} + +/* Structure used in the sql query to get configuration restore objects */ +struct plugin_config_handler_t +{ + UAContext *ua; /* UAContext for user input */ + POOLMEM *tmp; /* Used to store the config object */ + alist *plugins; /* Configuration plugin list */ + alist *content; /* Temp file used by each plugin */ +}; + +/* DB handler to get all configuration restore objects for a given + * set of jobids + */ +static int plugin_config_handler(void *ctx, int num_fields, char **row) +{ + struct plugin_config_handler_t *pch = (struct plugin_config_handler_t *)ctx; + UAContext *ua = pch->ua; + JCR *jcr = ua->jcr; + int32_t len; + + /* object */ + db_unescape_object(jcr, ua->db, + row[8], /* Object */ + str_to_uint64(row[1]), /* Object length */ + &pch->tmp, &len); + + /* Is compressed ? */ + if (str_to_int64(row[5]) > 0) { + int full_len = str_to_int64(row[2]); + int out_len = full_len + 100; /* full length */ + char *obj = (char *)malloc(out_len); + Zinflate(pch->tmp, len, obj, out_len); /* out_len is updated */ + if (out_len != full_len) { + ua->error_msg(_("Decompression failed. Len wanted=%d got=%d. Object=%s\n"), + full_len, out_len, row[9]); + } + obj[out_len] = 0; + pch->content->append(obj); + + } else { + pch->tmp[len]=0; + pch->content->append(bstrdup(pch->tmp)); + } + + pch->plugins->append(bstrdup(row[9])); + return 0; +} + +/* Save a Plugin Config object (ConfigFile) inside the JCR + * using a list of plugin_config_item + * + * We allow only one Plugin Config object per Plugin + */ +static void plugin_config_save_jcr(UAContext *ua, JCR *jcr, + char *pname, ConfigFile *ini) +{ + plugin_config_item *elt; + if (!jcr->plugin_config) { + jcr->plugin_config = New(alist(5, not_owned_by_alist)); + } + + /* Store only one Plugin Config object per plugin command */ + for (int i = 0; i < jcr->plugin_config->size() ; i++) { + elt = (plugin_config_item *) jcr->plugin_config->get(i); + if (strcmp(elt->plugin_name, pname) == 0) { + jcr->plugin_config->remove(i); + free_plugin_config_item(elt); + break; + } + } + + elt = (plugin_config_item *) malloc (sizeof(plugin_config_item)); + elt->plugin_name = bstrdup(pname); + elt->content = get_pool_memory(PM_FNAME); + ini->dump_results(&elt->content); + jcr->plugin_config->append(elt); +} + +/* TODO: Allow to have sub-menus Advanced.restore_mode can be + * in a Advanced panel (sub menu) + */ + +/* Take the ConfigIni struture and display user menu for a given plugin */ +static int plugin_display_options(UAContext *ua, JCR *jcr, ConfigFile *ini) +{ + int i, nb; + int jcr_pos = -1; + POOL_MEM prompt, tmp; + bool found; + INI_ITEM_HANDLER *h; + + /* TODO: See how to work in API mode + if (ua->api) { + ua->signal(BNET_RUN_CMD); + } + */ + + /* Take a look in the plugin_config list to see if we have something to + * initialize + */ + if (jcr->plugin_config) { + plugin_config_item *item=NULL; + + for (jcr_pos = 0; jcr_pos < jcr->plugin_config->size() ; jcr_pos++) { + item = (plugin_config_item *)jcr->plugin_config->get(jcr_pos); + + if (strcmp(item->plugin_name, ini->plugin_name) == 0) /* bpipe:xxx:yyyy */ + { + if (!ini->dump_string(item->content, strlen(item->content)) || + !ini->parse(ini->out_fname)) + { + ua->error_msg(_("Unable to use current plugin configuration, " + "discarding it.")); + } + /* When we are here, we can type yes (it will add it back), or no + * to not use this plugin configuration. So, don't keep it in the + * list. + */ + jcr->plugin_config->remove(jcr_pos); + free_plugin_config_item(item); + break; + } + } + } + +configure_again: + ua->send_msg(_("Plugin Restore Options\n")); + + for (nb=0; ini->items[nb].name; nb++) { + + if (ini->items[nb].found) { + /* When calling the handler, It will convert the value + * to a string representation in ini->edit + */ + ini->items[nb].handler(NULL, ini, &ini->items[nb]); + } else { + if (ini->items[nb].required) { + pm_strcpy(ini->edit, _("*None, but required*")); + + } else { + pm_strcpy(ini->edit, _("*None*")); + } + } + + Mmsg(tmp, "%s:", ini->items[nb].name); + + Mmsg(prompt, "%-20s %-20s ", + tmp.c_str(), ini->edit); + + if (ini->items[nb].default_value) { + Mmsg(tmp, "(%s)", ini->items[nb].default_value); + pm_strcat(prompt, tmp.c_str()); + } + + ua->send_msg("%s\n", prompt.c_str()); + } + + if (!get_cmd(ua, _("Use above plugin configuration? (yes/mod/no): "))) { + ini->clear_items(); + return 0; + } + + /* '', 'y', 'ye', and 'yes' are valid */ + if (strncasecmp(ua->cmd, _("yes"), strlen(ua->cmd)) == 0) { + return 1; + } + + if (strncasecmp(ua->cmd, _("no"), strlen(ua->cmd)) == 0) { + ini->clear_items(); + return 0; + } + + /* When using "mod", we display the list of parameters with their + * comments, and we let the user choose one entry to modify + */ + if (strncasecmp(ua->cmd, _("mod"), strlen(ua->cmd)) == 0) { + start_prompt(ua, _("You have the following choices:\n")); + + for (nb=0; ini->items[nb].name; nb++) { + + if (ini->items[nb].comment) { + Mmsg(tmp, " (%s)", ini->items[nb].comment); + } else { + pm_strcpy(tmp, ""); + } + + Mmsg(prompt, "%s%s ", + ini->items[nb].name, tmp.c_str()); + + add_prompt(ua, prompt.c_str()); + } + + i = do_prompt(ua, NULL, _("Select parameter to modify"), NULL, 0); + + if (i < 0) { + ini->clear_items(); + return 0; + } + + Mmsg(prompt, _("Please enter a value for %s: "), ini->items[i].name); + + /* Now use the handler to know how to ask the value to the user. + * For example, boolean will use get_yes_no(), pint32 will use get_pint() + */ + h = ini->items[i].handler; + if (h == ini_store_int32 || + h == ini_store_pint32) { + found = ini->items[i].found = get_pint(ua, prompt.c_str()); + if (found) { + ini->items[i].val.int32val = ua->pint32_val; + } + + } else if (h == ini_store_bool) { + found = ini->items[i].found = get_yesno(ua, prompt.c_str()); + if (found) { + ini->items[i].val.boolval = ua->pint32_val; + } + + } else if (h == ini_store_name) { + found = ini->items[i].found = get_cmd(ua, prompt.c_str()); + if (found) { + strncpy(ini->items[i].val.nameval, ua->cmd, MAX_NAME_LENGTH -1); + ini->items[i].val.nameval[MAX_NAME_LENGTH - 1] = 0; + } + + } else if (h == ini_store_str) { + found = ini->items[i].found = get_cmd(ua, prompt.c_str()); + if (found) { + ini->items[i].val.strval = bstrdup(ua->cmd); + } + + } else if (h == ini_store_int64 || + h == ini_store_pint64) { + found = ini->items[i].found = get_pint(ua, prompt.c_str()); + if (found) { + ini->items[i].val.int64val = ua->int64_val; + } + } + goto configure_again; + } + + return 1; /* never reached */ +} + +/* Display a menu with all plugins */ +static void plugin_config(UAContext *ua, JCR *jcr, run_ctx &rc) +{ + int i, nb; + char *elt, *tmp; + ConfigFile *ini = NULL; + POOLMEM *query=NULL; + struct plugin_config_handler_t pch; + + /* No jobids for this restore, probably wrong */ + if (!jcr->JobIds || !jcr->JobIds[0]) { + return; + } + + if (!open_client_db(ua)) { + return; + } + + pch.ua = ua; + query = get_pool_memory(PM_FNAME); + pch.tmp = get_pool_memory(PM_MESSAGE); + pch.plugins = New(alist(10, owned_by_alist)); + pch.content = New(alist(10, owned_by_alist)); + + /* Get all RestoreObject PLUGIN_CONFIG for the given Job */ + Mmsg(query, get_restore_objects, jcr->JobIds, FT_PLUGIN_CONFIG); + db_sql_query(ua->db, query, plugin_config_handler, &pch); + + if (!pch.plugins || pch.plugins->size() == 0) { + ua->info_msg(_("No plugin to configure\n")); + goto bail_out; + } + + /* TODO: Let see if we want to configure plugins that were given in command + * line. + */ + + start_prompt(ua, _("Plugins to configure:\n")); + + nb=0; + foreach_alist(elt, pch.plugins) { + nb++; + pm_strcpy(query, elt); + add_prompt(ua, query); + } + + i = do_prompt(ua, "", _("Select plugin to configure"), NULL, 0); + + if (i < 0) { + goto bail_out; + } + + + elt = (char *)pch.plugins->get(i); + ini = new ConfigFile(); + /* Try to read the plugin configuration, if error, loop to configure + * something else, or bail_out + */ + tmp = (char *)pch.content->get(i); + if (!ini->dump_string(tmp, strlen(tmp)) || /* Send the string to a file */ + !ini->unserialize(ini->out_fname)) { /* Read the file to initialize the ConfigFile */ + + ua->error_msg(_("Can't configure %32s\n"), elt); + goto bail_out; + } + + ini->set_plugin_name(elt); + + if (plugin_display_options(ua, jcr, ini)) { + ini->dump_results(&query); + Dmsg1(50, "plugin: %s\n", query); + + /* Save the plugin somewhere in the JCR */ + plugin_config_save_jcr(ua, jcr, elt, ini); + } + +bail_out: + free_pool_memory(pch.tmp); + free_pool_memory(query); + if (ini) { + delete ini; + } + delete pch.plugins; + delete pch.content; +} + int modify_job_parameters(UAContext *ua, JCR *jcr, run_ctx &rc) { int i, opt; @@ -724,26 +1138,35 @@ int modify_job_parameters(UAContext *ua, JCR *jcr, run_ctx &rc) break; case 4: /* Client */ - rc.client = select_client_resource(ua); + { + int32_t jt = rc.job ? rc.job->JobType : JT_SYSTEM; + rc.client = select_client_resource(ua, jt); if (rc.client) { jcr->client = rc.client; goto try_again; } + } break; case 5: /* When */ - if (!get_cmd(ua, _("Please enter desired start time as YYYY-MM-DD HH:MM:SS (return for now): "))) { + if (!get_cmd(ua, _("Please enter start time as a duration or YYYY-MM-DD HH:MM:SS or return for now: "))) { break; } if (ua->cmd[0] == 0) { jcr->sched_time = time(NULL); } else { + utime_t duration; jcr->sched_time = str_to_utime(ua->cmd); if (jcr->sched_time == 0) { - ua->send_msg(_("Invalid time, using current time.\n")); - jcr->sched_time = time(NULL); + if (duration_to_utime(ua->cmd, &duration)) { + jcr->sched_time = time(NULL) + duration; + } else { + ua->send_msg(_("Invalid time, using current time.\n")); + jcr->sched_time = time(NULL); + } } } + goto try_again; case 6: /* Priority */ @@ -781,7 +1204,7 @@ int modify_job_parameters(UAContext *ua, JCR *jcr, run_ctx &rc) } if (ua->cmd[0] != 0) { jcr->RestoreBootstrap = bstrdup(ua->cmd); - fd = fopen(jcr->RestoreBootstrap, "rb"); + fd = bfopen(jcr->RestoreBootstrap, "rb"); if (!fd) { berrno be; ua->send_msg(_("Warning cannot open %s: ERR=%s\n"), @@ -855,16 +1278,23 @@ int modify_job_parameters(UAContext *ua, JCR *jcr, run_ctx &rc) } goto try_again; case 12: - /* Plugin Options */ - //generate_plugin_event(jcr, bEventJobConfig, &rc); - if (!get_cmd(ua, _("Please Plugin Options string: "))) { - break; - } - if (jcr->plugin_options) { - free(jcr->plugin_options); - jcr->plugin_options = NULL; + + if (jcr->getJobType() == JT_RESTORE) { + plugin_config(ua, jcr, rc); + + } else { + //generate_plugin_event(jcr, bEventJobConfig, &rc); + + /* Plugin Options */ + if (!get_cmd(ua, _("Please Plugin Options string: "))) { + break; + } + if (jcr->plugin_options) { + free(jcr->plugin_options); + jcr->plugin_options = NULL; + } + jcr->plugin_options = bstrdup(ua->cmd); } - jcr->plugin_options = bstrdup(ua->cmd); goto try_again; case -1: /* error or cancel */ goto bail_out; @@ -881,6 +1311,30 @@ try_again: return 0; } + +/* Not a good idea to start a job with the Scratch pool. It creates all kind + * of recycling issues while the job is running. See Mantis #303 + */ +bool check_pool(int32_t JobType, int32_t JobLevel, POOL *pool, POOL *next_pool, + const char **name) +{ + if (JobType == JT_BACKUP) { + if (pool && strcmp(pool->name(), NT_("Scratch")) == 0) { + *name = NT_("Pool"); + return false; + } + } + /* The NextPool should also not be a Scratch pool */ + if (JobType == JT_MIGRATE || JobType == JT_COPY || + (JobType == JT_BACKUP && JobLevel == L_VIRTUAL_FULL)) { + if (next_pool && strcmp(next_pool->name(), NT_("Scratch")) == 0) { + *name = NT_("NextPool"); + return false; + } + } + return true; +} + /* * Put the run context that we have at this point into the JCR. * That allows us to re-ask for the run context. @@ -950,10 +1404,15 @@ static bool set_run_context_in_jcr(UAContext *ua, JCR *jcr, run_ctx &rc) } if (rc.when) { + utime_t duration; jcr->sched_time = str_to_utime(rc.when); if (jcr->sched_time == 0) { - ua->send_msg(_("Invalid time, using current time.\n")); - jcr->sched_time = time(NULL); + if (duration_to_utime(rc.when, &duration)) { + jcr->sched_time = time(NULL) + duration; + } else { + ua->send_msg(_("Invalid time, using current time.\n")); + jcr->sched_time = time(NULL); + } } rc.when = NULL; } @@ -974,6 +1433,15 @@ static bool set_run_context_in_jcr(UAContext *ua, JCR *jcr, run_ctx &rc) rc.plugin_options = NULL; } + if (rc.plugin_config) { + if (jcr->plugin_config) { + free_plugin_config_items(jcr->plugin_config); + delete jcr->plugin_config; + } + jcr->plugin_config = rc.plugin_config; + rc.plugin_config = NULL; + } + if (rc.replace) { jcr->replace = 0; for (i=0; ReplaceOptions[i].name; i++) { @@ -1028,7 +1496,7 @@ static bool set_run_context_in_jcr(UAContext *ua, JCR *jcr, run_ctx &rc) } rc.replace = ReplaceOptions[0].name; for (i=0; ReplaceOptions[i].name; i++) { - if ((int)ReplaceOptions[i].token == jcr->replace) { + if (ReplaceOptions[i].token == (int)jcr->replace) { rc.replace = ReplaceOptions[i].name; } } @@ -1071,6 +1539,15 @@ static bool set_run_context_in_jcr(UAContext *ua, JCR *jcr, run_ctx &rc) jcr->IgnoreDuplicateJobChecking = rc.ignoreduplicatecheck; } + /* Do not start a Backup job from the Scratch Pool */ + const char *name; + if (!check_pool(jcr->getJobType(), jcr->getJobLevel(), + rc.pool, rc.next_pool, &name)) { + ua->send_msg(_("%s \"Scratch\" not valid in Job \"%s\".\n"), + name, rc.job->name()); + return false; + } + return true; } @@ -1494,8 +1971,8 @@ static bool display_job_parameters(UAContext *ua, JCR *jcr, JOB *job, const char bstrutime(dt, sizeof(dt), jcr->sched_time), jcr->catalog->name(), jcr->JobPriority, - NPRTB(jcr->plugin_options)); - + (jcr->plugin_config && jcr->plugin_config->size() > 0) ? + _("User specified") : _("*None*")); } else { ua->send_msg(_("Run Restore job\n" "JobName: %s\n" @@ -1521,7 +1998,8 @@ static bool display_job_parameters(UAContext *ua, JCR *jcr, JOB *job, const char bstrutime(dt, sizeof(dt), jcr->sched_time), jcr->catalog->name(), jcr->JobPriority, - NPRTB(jcr->plugin_options)); + (jcr->plugin_config && jcr->plugin_config->size() > 0) ? + _("User specified") : _("*None*")); } } else { if (ua->api) { @@ -1551,8 +2029,8 @@ static bool display_job_parameters(UAContext *ua, JCR *jcr, JOB *job, const char bstrutime(dt, sizeof(dt), jcr->sched_time), jcr->catalog->name(), jcr->JobPriority, - NPRTB(jcr->plugin_options)); - + (jcr->plugin_config && jcr->plugin_config->size() > 0) ? + _("User specified") : _("*None*")); } else { ua->send_msg(_("Run Restore job\n" "JobName: %s\n" @@ -1578,7 +2056,8 @@ static bool display_job_parameters(UAContext *ua, JCR *jcr, JOB *job, const char bstrutime(dt, sizeof(dt), jcr->sched_time), jcr->catalog->name(), jcr->JobPriority, - NPRTB(jcr->plugin_options)); + (jcr->plugin_config && jcr->plugin_config->size() > 0) ? + _("User specified") : _("*None*")); } } @@ -1615,7 +2094,8 @@ static bool display_job_parameters(UAContext *ua, JCR *jcr, JOB *job, const char bstrutime(dt, sizeof(dt), jcr->sched_time), jcr->catalog->name(), jcr->JobPriority, - NPRTB(jcr->plugin_options)); + (jcr->plugin_config && jcr->plugin_config->size() > 0) ? + _("User specified") : _("*None*")); } break; case JT_COPY: @@ -1736,6 +2216,8 @@ static bool scan_run_command_line_arguments(UAContext *ua, run_ctx &rc) "job", /* 30 */ "mediatype", /* 31 */ "nextpool", /* 32 override next pool name */ + "fdcalled", /* 33 */ + NULL}; #define YES_POS 14 @@ -1755,7 +2237,7 @@ static bool scan_run_command_line_arguments(UAContext *ua, run_ctx &rc) rc.spool_data_set = false; rc.ignoreduplicatecheck = false; rc.comment = NULL; - rc.plugin_options = NULL; + free_plugin_config_items(rc.plugin_config); for (i=1; iargc; i++) { Dmsg2(800, "Doing arg %d = %s\n", i, ua->argk[i]); @@ -2016,6 +2498,9 @@ static bool scan_run_command_line_arguments(UAContext *ua, run_ctx &rc) rc.next_pool_name = ua->argv[i]; kw_ok = true; break; + case 33: /* fdcalled */ + kw_ok = true; + break; default: break; } diff --git a/bacula/src/dird/ua_select.c b/bacula/src/dird/ua_select.c index 3e39ba93a4..c55229bbd3 100644 --- a/bacula/src/dird/ua_select.c +++ b/bacula/src/dird/ua_select.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -170,6 +170,7 @@ int do_keyword_prompt(UAContext *ua, const char *msg, const char **list) */ STORE *select_storage_resource(UAContext *ua, bool unique) { + POOL_MEM tmp; char name[MAX_NAME_LENGTH]; STORE *store; @@ -180,9 +181,10 @@ STORE *select_storage_resource(UAContext *ua, bool unique) start_prompt(ua, _("The defined Storage resources are:\n")); LockRes(); foreach_res(store, R_STORAGE) { - if (acl_access_ok(ua, Storage_ACL, store->name())) { + if (store->is_enabled() && acl_access_ok(ua, Storage_ACL, store->name())) { if (unique) { - add_prompt(ua, store->name(), store->address); + Mmsg(tmp, "%s:%d", store->address, store->SDport); + add_prompt(ua, store->name(), tmp.c_str()); } else { add_prompt(ua, store->name()); } @@ -238,7 +240,7 @@ CAT *get_catalog_resource(UAContext *ua) } } if (strcasecmp(ua->argk[i], NT_("client")) == 0 && ua->argv[i]) { - if (acl_access_ok(ua, Client_ACL, ua->argv[i])) { + if (acl_access_client_ok(ua, ua->argv[i], JT_BACKUP_RESTORE)) { client = (CLIENT *)GetResWithName(R_CLIENT, ua->argv[i]); break; } @@ -296,7 +298,7 @@ JOB *select_enable_disable_job_resource(UAContext *ua, bool enable) if (!acl_access_ok(ua, Job_ACL, job->name())) { continue; } - if (job->enabled == enable) { /* Already enabled/disabled? */ + if (job->is_enabled() == enable) { /* Already enabled/disabled? */ continue; /* yes, skip */ } add_prompt(ua, job->name()); @@ -320,7 +322,7 @@ JOB *select_job_resource(UAContext *ua) start_prompt(ua, _("The defined Job resources are:\n")); LockRes(); foreach_res(job, R_JOB) { - if (acl_access_ok(ua, Job_ACL, job->name())) { + if (job->is_enabled() && acl_access_ok(ua, Job_ACL, job->name())) { add_prompt(ua, job->name()); } } @@ -361,7 +363,8 @@ JOB *select_restore_job_resource(UAContext *ua) start_prompt(ua, _("The defined Restore Job resources are:\n")); LockRes(); foreach_res(job, R_JOB) { - if (job->JobType == JT_RESTORE && acl_access_ok(ua, Job_ACL, job->name())) { + if (job->JobType == JT_RESTORE && job->is_enabled() && + acl_access_ok(ua, Job_ACL, job->name())) { add_prompt(ua, job->name()); } } @@ -384,10 +387,10 @@ CLIENT *select_enable_disable_client_resource(UAContext *ua, bool enable) LockRes(); start_prompt(ua, _("The defined Client resources are:\n")); foreach_res(client, R_CLIENT) { - if (!acl_access_ok(ua, Client_ACL, client->name())) { + if (!acl_access_client_ok(ua, client->name(), JT_BACKUP_RESTORE)) { continue; } - if (client->enabled == enable) { /* Already enabled/disabled? */ + if (client->is_enabled() == enable) { /* Already enabled/disabled? */ continue; /* yes, skip */ } add_prompt(ua, client->name()); @@ -404,7 +407,7 @@ CLIENT *select_enable_disable_client_resource(UAContext *ua, bool enable) /* * Select a client resource from prompt list */ -CLIENT *select_client_resource(UAContext *ua) +CLIENT *select_client_resource(UAContext *ua, int32_t jobtype) { char name[MAX_NAME_LENGTH]; CLIENT *client; @@ -412,7 +415,7 @@ CLIENT *select_client_resource(UAContext *ua) start_prompt(ua, _("The defined Client resources are:\n")); LockRes(); foreach_res(client, R_CLIENT) { - if (acl_access_ok(ua, Client_ACL, client->name())) { + if (client->is_enabled() && acl_access_client_ok(ua, client->name(), jobtype)) { add_prompt(ua, client->name()); } } @@ -429,7 +432,7 @@ CLIENT *select_client_resource(UAContext *ua) * client= * if we don't find the keyword, we prompt the user. */ -CLIENT *get_client_resource(UAContext *ua) +CLIENT *get_client_resource(UAContext *ua, int32_t jobtype) { CLIENT *client = NULL; int i; @@ -437,7 +440,7 @@ CLIENT *get_client_resource(UAContext *ua) for (i=1; iargc; i++) { if ((strcasecmp(ua->argk[i], NT_("client")) == 0 || strcasecmp(ua->argk[i], NT_("fd")) == 0) && ua->argv[i]) { - if (!acl_access_ok(ua, Client_ACL, ua->argv[i])) { + if (!acl_access_client_ok(ua, ua->argv[i], jobtype)) { break; } client = (CLIENT *)GetResWithName(R_CLIENT, ua->argv[i]); @@ -448,7 +451,7 @@ CLIENT *get_client_resource(UAContext *ua) break; } } - return select_client_resource(ua); + return select_client_resource(ua, jobtype); } /* @@ -465,7 +468,7 @@ SCHED *select_enable_disable_schedule_resource(UAContext *ua, bool enable) if (!acl_access_ok(ua, Schedule_ACL, sched->name())) { continue; } - if (sched->enabled == enable) { /* Already enabled/disabled? */ + if (sched->is_enabled() == enable) { /* Already enabled/disabled? */ continue; /* yes, skip */ } add_prompt(ua, sched->name()); @@ -489,7 +492,7 @@ SCHED *select_enable_disable_schedule_resource(UAContext *ua, bool enable) * returns: 0 on error * 1 on success and fills in CLIENT_DBR */ -bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr) +bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr, int32_t jobtype) { int i; @@ -502,7 +505,7 @@ bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr) for (i=1; iargc; i++) { if ((strcasecmp(ua->argk[i], NT_("client")) == 0 || strcasecmp(ua->argk[i], NT_("fd")) == 0) && ua->argv[i]) { - if (!acl_access_ok(ua, Client_ACL, ua->argv[i])) { + if (!acl_access_client_ok(ua, ua->argv[i], jobtype)) { break; } bstrncpy(cr->Name, ua->argv[i], sizeof(cr->Name)); @@ -515,7 +518,7 @@ bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr) return 1; } } - if (!select_client_dbr(ua, cr)) { /* try once more by proposing a list */ + if (!select_client_dbr(ua, cr, jobtype)) { /* try once more by proposing a list */ return 0; } return 1; @@ -526,7 +529,7 @@ bool get_client_dbr(UAContext *ua, CLIENT_DBR *cr) * Returns 1 on success * 0 on failure */ -bool select_client_dbr(UAContext *ua, CLIENT_DBR *cr) +bool select_client_dbr(UAContext *ua, CLIENT_DBR *cr, int32_t jobtype) { CLIENT_DBR ocr; char name[MAX_NAME_LENGTH]; @@ -548,7 +551,7 @@ bool select_client_dbr(UAContext *ua, CLIENT_DBR *cr) for (i=0; i < num_clients; i++) { ocr.ClientId = ids[i]; if (!db_get_client_record(ua->jcr, ua->db, &ocr) || - !acl_access_ok(ua, Client_ACL, ocr.Name)) { + !acl_access_client_ok(ua, ocr.Name, jobtype)) { continue; } add_prompt(ua, ocr.Name); @@ -1097,12 +1100,13 @@ done: */ STORE *get_storage_resource(UAContext *ua, bool use_default, bool unique) { - char *store_name = NULL; + char store_name[MAX_NAME_LENGTH]; STORE *store = NULL; int jobid; JCR *jcr; int i; char ed1[50]; + *store_name = 0; for (i=1; iargc; i++) { if (use_default && !ua->argv[i]) { @@ -1113,21 +1117,20 @@ STORE *get_storage_resource(UAContext *ua, bool use_default, bool unique) strcasecmp("slots", ua->argk[i]) == 0) { continue; } - /* Default argument is storage */ - if (store_name) { + /* Default argument is storage (except in enable/disable command) */ + if (store_name[0]) { ua->error_msg(_("Storage name given twice.\n")); return NULL; } - store_name = ua->argk[i]; - if (*store_name == '?') { + bstrncpy(store_name, ua->argk[i], sizeof(store_name)); + if (store_name[0] == '?') { *store_name = 0; break; } } else { if (strcasecmp(ua->argk[i], NT_("storage")) == 0 || strcasecmp(ua->argk[i], NT_("sd")) == 0) { - store_name = ua->argv[i]; - break; + bstrncpy(store_name, NPRTB(ua->argv[i]), sizeof(store_name)); } else if (strcasecmp(ua->argk[i], NT_("jobid")) == 0) { jobid = str_to_int64(ua->argv[i]); @@ -1139,9 +1142,10 @@ STORE *get_storage_resource(UAContext *ua, bool use_default, bool unique) ua->error_msg(_("JobId %s is not running.\n"), edit_int64(jobid, ed1)); return NULL; } - store = jcr->wstore; + if (jcr->wstore) { + bstrncpy(store_name, jcr->wstore->name(), sizeof(store_name)); + } free_jcr(jcr); - break; } else if (strcasecmp(ua->argk[i], NT_("job")) == 0 || strcasecmp(ua->argk[i], NT_("jobname")) == 0) { @@ -1153,32 +1157,35 @@ STORE *get_storage_resource(UAContext *ua, bool use_default, bool unique) ua->error_msg(_("Job \"%s\" is not running.\n"), ua->argv[i]); return NULL; } - store = jcr->wstore; + if (jcr->wstore) { + bstrncpy(store_name, jcr->wstore->name(), sizeof(store_name)); + } free_jcr(jcr); - break; + } else if (strcasecmp(ua->argk[i], NT_("ujobid")) == 0) { if (!ua->argv[i]) { ua->error_msg(_("Expecting ujobid=xxx, got: %s.\n"), ua->argk[i]); return NULL; } if ((jcr=get_jcr_by_full_name(ua->argv[i]))) { - store = jcr->wstore; - free_jcr(jcr); - /* The job might not be running, so we try to see other keywords */ - if (store) { - break; + if (jcr->wstore) { + bstrncpy(store_name, jcr->wstore->name(), sizeof(store_name)); } + free_jcr(jcr); } } + if (store_name[0]) { + break; /* We can stop the loop if we have something */ + } } } - if (store && !acl_access_ok(ua, Storage_ACL, store->name())) { - store = NULL; - } - if (!store && store_name && store_name[0] != 0) { + if (store_name[0] != 0) { store = (STORE *)GetResWithName(R_STORAGE, store_name); - if (!store) { + if (!store && strcmp(store_name, "storage") != 0) { + /* Looks that the first keyword of the line was not a storage name, make + * sure that it's not "storage=" before we print the following message + */ ua->error_msg(_("Storage resource \"%s\": not found\n"), store_name); } } @@ -1265,25 +1272,37 @@ int get_media_type(UAContext *ua, char *MediaType, int max_media) start_prompt(ua, _("Media Types defined in conf file:\n")); LockRes(); foreach_res(store, R_STORAGE) { - add_prompt(ua, store->media_type); + if (store->is_enabled()) { + add_prompt(ua, store->media_type); + } } UnlockRes(); return (do_prompt(ua, _("Media Type"), _("Select the Media Type"), MediaType, max_media) < 0) ? 0 : 1; } - -bool get_level_from_name(JCR *jcr, const char *level_name) +int get_level_code_from_name(const char *level_name) { - /* Look up level name and pull code */ - bool found = false; + int ret = 0; + if (!level_name) { + return ret; + } for (int i=0; joblevels[i].level_name; i++) { if (strcasecmp(level_name, joblevels[i].level_name) == 0) { - jcr->setJobLevel(joblevels[i].level); - found = true; + ret = joblevels[i].level; break; } } - return found; + return ret; +} + +bool get_level_from_name(JCR *jcr, const char *level_name) +{ + int level = get_level_code_from_name(level_name); + if (level > 0) { + jcr->setJobLevel(level); + return true; + } + return false; } static int count_running_jobs(UAContext *ua) @@ -1496,3 +1515,134 @@ bail_out: if (selected) delete selected; return jcrs->size(); } + +/* Small helper to scan storage daemon commands and search for volumes */ +int scan_storage_cmd(UAContext *ua, const char *cmd, + bool allfrompool, /* Choose to select a specific volume or not */ + int *drive, /* Drive number */ + MEDIA_DBR *mr, /* Media Record, can have options already filled */ + POOL_DBR *pr, /* Pool Record */ + const char **action, /* action= argument, can be NULL if not relevant */ + char *storage, /* Storage name, must be MAX_NAME_LENGTH long */ + int *nb, /* Number of media found */ + uint32_t **results) /* List of MediaId */ +{ + bool allpools=false, has_vol = false;; + STORE *store; + + *nb = 0; + *drive = 0; + *results = NULL; + *storage = 0; + + /* Look at arguments */ + for (int i=1; iargc; i++) { + if (strcasecmp(ua->argk[i], NT_("allpools")) == 0) { + allpools = true; + + } else if (strcasecmp(ua->argk[i], NT_("allfrompool")) == 0) { + allfrompool = true; + + } else if (strcasecmp(ua->argk[i], NT_("volume")) == 0 + && is_name_valid(ua->argv[i], NULL)) { + bstrncpy(mr->VolumeName, ua->argv[i], sizeof(mr->VolumeName)); + has_vol = true; + + } else if (strcasecmp(ua->argk[i], NT_("mediatype")) == 0 + && ua->argv[i]) { + bstrncpy(mr->MediaType, ua->argv[i], sizeof(mr->MediaType)); + + } else if (strcasecmp(ua->argk[i], NT_("drive")) == 0 && ua->argv[i]) { + *drive = atoi(ua->argv[i]); + + } else if (strcasecmp(ua->argk[i], NT_("action")) == 0 + && is_name_valid(ua->argv[i], NULL)) { + + if (action) { + *action = ua->argv[i]; + } else { + ua->warning_msg(_("Invalid argument \"action\".\n")); + } + } + } + + /* Choose storage */ + ua->jcr->wstore = store = get_storage_resource(ua, false); + if (!store) { + goto bail_out; + } + bstrncpy(storage, store->dev_name(), MAX_NAME_LENGTH); + + if (!open_db(ua)) { + Dmsg0(100, "Can't open db\n"); + goto bail_out; + } + + /* + * Look for all volumes that are enabled + */ + mr->Enabled = 1; + set_storageid_in_mr(store, mr); + + if (allfrompool && !has_vol) { /* We need a list of volumes */ + + /* We don't take all pools and we don't have a volume in argument, + * so we need to choose a pool + */ + if (!allpools) { + /* force pool selection */ + POOL *pool = get_pool_resource(ua); + + if (!pool) { + Dmsg0(100, "Can't get pool resource\n"); + goto bail_out; + } + bstrncpy(pr->Name, pool->name(), sizeof(pr->Name)); + if (!db_get_pool_record(ua->jcr, ua->db, pr)) { + Dmsg0(100, "Can't get pool record\n"); + goto bail_out; + } + mr->PoolId = pr->PoolId; + } + + if (!db_get_media_ids(ua->jcr, ua->db, mr, nb, results)) { + Dmsg0(100, "No results from db_get_media_ids\n"); + goto bail_out; + } + + } else { /* We want a single volume */ + MEDIA_DBR mr2; + if (!select_media_dbr(ua, &mr2)) { + goto bail_out; + } + /* The select_media_dbr() doesn't filter on the CacheRetention */ + if (mr->CacheRetention) { + if ((mr2.CacheRetention + mr2.LastWritten) > time(NULL)) { + /* The volume cache retention is not exipred */ + goto bail_out; + } + } + *nb = 1; + *results = (uint32_t *) malloc(1 * sizeof(uint32_t)); + *results[0] = mr2.MediaId; + } + + if (*nb == 0) { + goto bail_out; + } + return 1; + +bail_out: + if (!*nb) { + ua->send_msg(_("No Volumes found to perform the command.\n")); + } + + close_db(ua); + ua->jcr->wstore = NULL; + if (*results) { + free(*results); + *results = NULL; + } + *nb = 0; + return 0; +} diff --git a/bacula/src/dird/ua_status.c b/bacula/src/dird/ua_status.c index 4f160308db..97b25374d4 100644 --- a/bacula/src/dird/ua_status.c +++ b/bacula/src/dird/ua_status.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -17,11 +17,9 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Status Command * * Kern Sibbald, August MMI - * */ @@ -96,7 +94,7 @@ bool dot_status_cmd(UAContext *ua, const char *cmd) return false; } } else if (strcasecmp(ua->argk[1], "client") == 0) { - client = get_client_resource(ua); + client = get_client_resource(ua, JT_BACKUP_RESTORE); if (client) { Dmsg2(200, "Client=%s arg=%s\n", client->name(), NPRT(ua->argk[2])); do_client_status(ua, client, ua->argk[2]); @@ -116,6 +114,132 @@ bool dot_status_cmd(UAContext *ua, const char *cmd) return true; } +/* Test the network between FD and SD */ +static int do_network_status(UAContext *ua) +{ + CLIENT *client = NULL; + USTORE store; + JCR *jcr = ua->jcr; + char *store_address, ed1[50]; + uint32_t store_port; + uint64_t nb = 50 * 1024 * 1024; + + int i = find_arg_with_value(ua, "bytes"); + if (i > 0) { + if (!size_to_uint64(ua->argv[i], strlen(ua->argv[i]), &nb)) { + return 1; + } + } + + client = get_client_resource(ua, JT_BACKUP_RESTORE); + if (!client) { + return 1; + } + + store.store = get_storage_resource(ua, false, true); + if (!store.store) { + return 1; + } + + jcr->client = client; + set_wstorage(jcr, &store); + + if (!ua->api) { + ua->send_msg(_("Connecting to Storage %s at %s:%d\n"), + store.store->name(), store.store->address, store.store->SDport); + } + + if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) { + ua->error_msg(_("Failed to connect to Storage.\n")); + goto bail_out; + } + + if (!start_storage_daemon_job(jcr, NULL, NULL)) { + goto bail_out; + } + + /* + * Note startup sequence of SD/FD is different depending on + * whether the SD listens (normal) or the SD calls the FD. + */ + if (!client->sd_calls_client) { + if (!run_storage_and_start_message_thread(jcr, jcr->store_bsock)) { + goto bail_out; + } + } /* Else it's done in init_storage_job() */ + + if (!ua->api) { + ua->send_msg(_("Connecting to Client %s at %s:%d\n"), + client->name(), client->address(), client->FDport); + } + + if (!connect_to_file_daemon(jcr, 1, 15, 0)) { + ua->error_msg(_("Failed to connect to Client.\n")); + goto bail_out; + } + + if (jcr->sd_calls_client) { + /* + * SD must call "client" i.e. FD + */ + if (jcr->FDVersion < 10) { + Jmsg(jcr, M_FATAL, 0, _("The File daemon does not support SDCallsClient.\n")); + goto bail_out; + } + if (!send_client_addr_to_sd(jcr)) { + goto bail_out; + } + if (!run_storage_and_start_message_thread(jcr, jcr->store_bsock)) { + goto bail_out; + } + + store_address = store.store->address; /* dummy */ + store_port = 0; /* flag that SD calls FD */ + + } else { + /* + * send Storage daemon address to the File daemon, + * then wait for File daemon to make connection + * with Storage daemon. + */ + if (store.store->SDDport == 0) { + store.store->SDDport = store.store->SDport; + } + + store_address = get_storage_address(jcr->client, store.store); + store_port = store.store->SDDport; + } + + if (!send_store_addr_to_fd(jcr, store.store, store_address, store_port)) { + goto bail_out; + } + + if (!ua->api) { + ua->info_msg(_("Running network test between Client=%s and Storage=%s with %sB ...\n"), + client->name(), store.store->name(), edit_uint64_with_suffix(nb, ed1)); + } + + if (!jcr->file_bsock->fsend("testnetwork bytes=%lld\n", nb)) { + goto bail_out; + } + + while (jcr->file_bsock->recv() > 0) { + ua->info_msg(jcr->file_bsock->msg); + } + +bail_out: + jcr->file_bsock->signal(BNET_TERMINATE); + jcr->store_bsock->signal(BNET_TERMINATE); + wait_for_storage_daemon_termination(jcr); + + free_bsock(jcr->file_bsock); + free_bsock(jcr->store_bsock); + + jcr->client = NULL; + free_wstorage(jcr); + return 1; +} + /* This is the *old* command handler, so we must return * 1 or it closes the connection */ @@ -137,7 +261,10 @@ int status_cmd(UAContext *ua, const char *cmd) Dmsg1(20, "status:%s:\n", cmd); for (i=1; iargc; i++) { - if (strcasecmp(ua->argk[i], NT_("schedule")) == 0 || + if (strcasecmp(ua->argk[i], NT_("network")) == 0) { + do_network_status(ua); + return 1; + } else if (strcasecmp(ua->argk[i], NT_("schedule")) == 0 || strcasecmp(ua->argk[i], NT_("scheduled")) == 0) { llist_scheduled_jobs(ua); return 1; @@ -149,7 +276,7 @@ int status_cmd(UAContext *ua, const char *cmd) do_director_status(ua); return 1; } else if (strcasecmp(ua->argk[i], NT_("client")) == 0) { - client = get_client_resource(ua); + client = get_client_resource(ua, JT_BACKUP_RESTORE); if (client) { do_client_status(ua, client, NULL); } @@ -175,6 +302,7 @@ int status_cmd(UAContext *ua, const char *cmd) add_prompt(ua, NT_("Storage")); add_prompt(ua, NT_("Client")); add_prompt(ua, NT_("Scheduled")); + add_prompt(ua, NT_("Network")); add_prompt(ua, NT_("All")); Dmsg0(20, "do_prompt: select daemon\n"); if ((item=do_prompt(ua, "", _("Select daemon type for status"), prmt, sizeof(prmt))) < 0) { @@ -192,7 +320,7 @@ int status_cmd(UAContext *ua, const char *cmd) } break; case 2: - client = select_client_resource(ua); + client = select_client_resource(ua, JT_BACKUP_RESTORE); if (client) { do_client_status(ua, client, NULL); } @@ -201,6 +329,9 @@ int status_cmd(UAContext *ua, const char *cmd) llist_scheduled_jobs(ua); break; case 4: + do_network_status(ua); + break; + case 5: do_all_status(ua); break; default: @@ -264,11 +395,11 @@ static void do_all_status(UAContext *ua) i = 0; foreach_res(client, R_CLIENT) { found = false; - if (!acl_access_ok(ua, Client_ACL, client->name())) { + if (!acl_access_client_ok(ua, client->name(), JT_BACKUP_RESTORE)) { continue; } for (j=0; jaddress, client->address) == 0 && + if (strcmp(unique_client[j]->address(), client->address()) == 0 && unique_client[j]->FDport == client->FDport) { found = true; break; @@ -276,7 +407,7 @@ static void do_all_status(UAContext *ua) } if (!found) { unique_client[i++] = client; - Dmsg2(40, "Stuffing: %s:%d\n", client->address, client->FDport); + Dmsg2(40, "Stuffing: %s:%d\n", client->address(), client->FDport); } } UnlockRes(); @@ -289,24 +420,63 @@ static void do_all_status(UAContext *ua) } +static void api_list_dir_status_header(UAContext *ua) +{ + OutputWriter wt(ua->api_opts); + wt.start_group("header"); + wt.get_output( + OT_STRING, "name", my_name, + OT_STRING, "version", VERSION " (" BDATE ")", + OT_STRING, "uname", HOST_OS " " DISTNAME " " DISTVER, + OT_UTIME, "started", daemon_start_time, + OT_UTIME, "reloaded", last_reload_time, + OT_INT, "jobs_run", num_jobs_run, + OT_INT, "jobs_running",job_count(), + OT_INT, "nclients", ((rblist *)res_head[R_CLIENT-r_first]->res_list)->size(), + OT_INT, "nstores", ((rblist *)res_head[R_STORAGE-r_first]->res_list)->size(), + OT_INT, "npools", ((rblist *)res_head[R_POOL-r_first]->res_list)->size(), + OT_INT, "ncats", ((rblist *)res_head[R_CATALOG-r_first]->res_list)->size(), + OT_INT, "nfset", ((rblist *)res_head[R_FILESET-r_first]->res_list)->size(), + OT_INT, "nscheds", ((rblist *)res_head[R_SCHEDULE-r_first]->res_list)->size(), + OT_PLUGINS,"plugins", b_plugin_list, + OT_END); + + ua->send_msg("%s", wt.end_group()); +} + void list_dir_status_header(UAContext *ua) { char dt[MAX_TIME_LENGTH], dt1[MAX_TIME_LENGTH]; char b1[35], b2[35], b3[35], b4[35], b5[35]; + if (ua->api > 1) { + api_list_dir_status_header(ua); + return; + } + ua->send_msg(_("%s %sVersion: %s (%s) %s %s %s\n"), my_name, "", VERSION, BDATE, HOST_OS, DISTNAME, DISTVER); bstrftime_nc(dt, sizeof(dt), daemon_start_time); bstrftimes(dt1, sizeof(dt1), last_reload_time); ua->send_msg(_("Daemon started %s, conf reloaded %s\n"), dt, dt1); - ua->send_msg(_(" Jobs: run=%d, running=%d mode=%d\n"), - num_jobs_run, job_count(), (int)DEVELOPER_MODE); + ua->send_msg(_(" Jobs: run=%d, running=%d mode=%d,%d\n"), + num_jobs_run, job_count(), (int)DEVELOPER_MODE, 0); ua->send_msg(_(" Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n"), edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1), edit_uint64_with_commas(sm_bytes, b2), edit_uint64_with_commas(sm_max_bytes, b3), edit_uint64_with_commas(sm_buffers, b4), edit_uint64_with_commas(sm_max_buffers, b5)); + ua->send_msg(_(" Res: njobs=%d nclients=%d nstores=%d npools=%d ncats=%d" + " nfsets=%d nscheds=%d\n"), + ((rblist *)res_head[R_JOB-r_first]->res_list)->size(), + ((rblist *)res_head[R_CLIENT-r_first]->res_list)->size(), + ((rblist *)res_head[R_STORAGE-r_first]->res_list)->size(), + ((rblist *)res_head[R_POOL-r_first]->res_list)->size(), + ((rblist *)res_head[R_CATALOG-r_first]->res_list)->size(), + ((rblist *)res_head[R_FILESET-r_first]->res_list)->size(), + ((rblist *)res_head[R_SCHEDULE-r_first]->res_list)->size()); + /* TODO: use this function once for all daemons */ if (b_plugin_list && b_plugin_list->size() > 0) { @@ -421,7 +591,7 @@ static void do_client_status(UAContext *ua, CLIENT *client, char *cmd) { BSOCK *fd; - if (!acl_access_ok(ua, Client_ACL, client->name())) { + if (!acl_access_client_ok(ua, client->name(), JT_BACKUP_RESTORE)) { ua->error_msg(_("No authorization for Client \"%s\"\n"), client->name()); return; } @@ -436,7 +606,7 @@ static void do_client_status(UAContext *ua, CLIENT *client, char *cmd) /* Try to connect for 15 seconds */ if (!ua->api) ua->send_msg(_("Connecting to Client %s at %s:%d\n"), - client->name(), client->address, client->FDport); + client->name(), client->address(), client->FDport); if (!connect_to_file_daemon(ua->jcr, 1, 15, 0)) { ua->send_msg(_("Failed to connect to Client %s.\n====\n"), client->name()); @@ -489,7 +659,7 @@ struct sched_pkt { STORE *store; }; -static void prt_runtime(UAContext *ua, sched_pkt *sp) +static void prt_runtime(UAContext *ua, sched_pkt *sp, OutputWriter *ow) { char dt[MAX_TIME_LENGTH]; const char *level_ptr; @@ -535,6 +705,23 @@ static void prt_runtime(UAContext *ua, sched_pkt *sp) ua->send_msg(_("%-14s\t%-8s\t%3d\t%-18s\t%-18s\t%s\n"), level_ptr, job_type_to_str(sp->job->JobType), sp->priority, dt, sp->job->name(), mr.VolumeName); + + } else if (ua->api > 1) { + ua->send_msg("%s", + ow->get_output(OT_CLEAR, + OT_START_OBJ, + OT_STRING, "name", sp->job->name(), + OT_JOBLEVEL, "level", sp->level, + OT_JOBTYPE, "type", sp->job->JobType, + OT_INT, "priority",sp->priority, + OT_UTIME, "schedtime", sp->runtime, + OT_STRING, "volume", mr.VolumeName, + OT_STRING, "pool", jcr->pool?jcr->pool->name():"", + OT_STRING, "storage", jcr->wstore?jcr->wstore->name():"", + OT_END_OBJ, + OT_END)); + + } else { ua->send_msg(_("%-14s %-8s %3d %-18s %-18s %s\n"), level_ptr, job_type_to_str(sp->job->JobType), sp->priority, dt, @@ -617,8 +804,8 @@ static void llist_scheduled_jobs(UAContext *ua) LockRes(); foreach_res(job, R_JOB) { sched = job->schedule; - if (!sched || !job->enabled || (sched && !sched->enabled) || - (job->client && !job->client->enabled)) { + if (!sched || !job->is_enabled() || (sched && !sched->is_enabled()) || + (job->client && !job->client->is_enabled())) { continue; /* no, skip this job */ } if (job_name[0] && strcmp(job_name, job->name()) != 0) { @@ -752,6 +939,7 @@ static int my_compare(void *item1, void *item2) */ static void list_scheduled_jobs(UAContext *ua) { + OutputWriter ow(ua->api_opts); utime_t runtime; RUN *run; JOB *job; @@ -784,7 +972,7 @@ static void list_scheduled_jobs(UAContext *ua) /* Loop through all jobs */ LockRes(); foreach_res(job, R_JOB) { - if (!acl_access_ok(ua, Job_ACL, job->name()) || !job->enabled) { + if (!acl_access_ok(ua, Job_ACL, job->name()) || !job->is_enabled()) { continue; } if (sched_name[0] && job->schedule && @@ -820,7 +1008,7 @@ static void list_scheduled_jobs(UAContext *ua) } /* end for loop over resources */ UnlockRes(); foreach_dlist(sp, &sched) { - prt_runtime(ua, sp); + prt_runtime(ua, sp, &ow); } if (num_jobs == 0 && !ua->api) { ua->send_msg(_("No Scheduled Jobs.\n")); @@ -834,11 +1022,13 @@ static void list_running_jobs(UAContext *ua) JCR *jcr; int njobs = 0; int i; - const char *msg; + int32_t status; + const char *msg, *msgdir; char *emsg; /* edited message */ char dt[MAX_TIME_LENGTH]; char level[10]; bool pool_mem = false; + OutputWriter ow(ua->api_opts); JobId_t jid = 0; if ((i = find_arg_with_value(ua, "jobid")) >= 0) { @@ -862,30 +1052,32 @@ static void list_running_jobs(UAContext *ua) } continue; } - njobs++; } endeach_jcr(jcr); - if (njobs == 0) { - /* Note the following message is used in regress -- don't change */ - ua->send_msg(_("No Jobs running.\n====\n")); - Dmsg0(200, "leave list_run_jobs()\n"); - return; - } } - if (!ua->api) { - ua->send_msg(_(" JobId Type Level Files Bytes Name Status\n")); - ua->send_msg(_("======================================================================\n")); - } + njobs = 0; /* count the number of job really displayed */ foreach_jcr(jcr) { - if (jcr->JobId == 0 || !acl_access_ok(ua, Job_ACL, jcr->job->name())) { + if (jcr->JobId == 0 || !jcr->job || !acl_access_ok(ua, Job_ACL, jcr->job->name())) { continue; } /* JobId keyword found in command line */ if (jid > 0 && jcr->JobId != jid) { continue; } - switch (jcr->JobStatus) { + + if (++njobs == 1) { + /* display the header for the first job */ + if (!ua->api) { + ua->send_msg(_(" JobId Type Level Files Bytes Name Status\n")); + ua->send_msg(_("======================================================================\n")); + + } else if (ua->api > 1) { + ua->send_msg(ow.start_group("running", false)); + } + } + status = jcr->JobStatus; + switch (status) { case JS_Created: msg = _("is waiting execution"); break; @@ -986,6 +1178,7 @@ static void list_running_jobs(UAContext *ua) msg = emsg; break; } + msgdir = msg; /* Keep it to know if we update the status variable */ /* * Now report Storage daemon status code */ @@ -1034,6 +1227,9 @@ static void list_running_jobs(UAContext *ua) msg = _("Dir inserting Attributes"); break; } + if (msg != msgdir) { + status = jcr->SDJobStatus; + } switch (jcr->getJobType()) { case JT_ADMIN: bstrncpy(level, "Admin", sizeof(level)); @@ -1053,6 +1249,30 @@ static void list_running_jobs(UAContext *ua) jcr->JobId, level, jcr->Job, msg, jcr->comment); unbash_spaces(jcr->comment); + } else if (ua->api > 1) { + ua->send_msg("%s", ow.get_output(OT_CLEAR, + OT_START_OBJ, + OT_INT32, "jobid", jcr->JobId, + OT_JOBLEVEL,"level", jcr->getJobLevel(), + OT_JOBTYPE, "type", jcr->getJobType(), + OT_JOBSTATUS,"status", status, + OT_STRING, "status_desc",msg, + OT_STRING, "comment", jcr->comment, + OT_SIZE, "jobbytes", jcr->JobBytes, + OT_INT32, "jobfiles", jcr->JobFiles, + OT_STRING, "job", jcr->Job, + OT_STRING, "name", jcr->job->name(), + OT_STRING, "clientname",jcr->client?jcr->client->name():"", + OT_STRING, "fileset", jcr->fileset?jcr->fileset->name():"", + OT_STRING, "storage", jcr->wstore?jcr->wstore->name():"", + OT_STRING, "rstorage", jcr->rstore?jcr->rstore->name():"", + OT_UTIME, "schedtime", jcr->sched_time, + OT_UTIME, "starttime", jcr->start_time, + OT_INT32, "priority", jcr->JobPriority, + OT_INT32, "errors", jcr->JobErrors, + OT_END_OBJ, + OT_END)); + } else { char b1[50], b2[50], b3[50]; level[4] = 0; @@ -1071,8 +1291,19 @@ static void list_running_jobs(UAContext *ua) } } endeach_jcr(jcr); - if (!ua->api) { - ua->send_msg("====\n"); + + if (njobs == 0) { + /* Note the following message is used in regress -- don't change */ + ua->send_msg(_("No Jobs running.\n====\n")); + Dmsg0(200, "leave list_run_jobs()\n"); + return; + } else { + /* display a closing header */ + if (!ua->api) { + ua->send_msg("====\n"); + } else if (ua->api > 1) { + ua->send_msg(ow.end_group(false)); + } } Dmsg0(200, "leave list_run_jobs()\n"); } @@ -1081,6 +1312,7 @@ static void list_terminated_jobs(UAContext *ua) { char dt[MAX_TIME_LENGTH], b1[30], b2[30]; char level[10]; + OutputWriter ow(ua->api_opts); if (last_jobs->empty()) { if (!ua->api) ua->send_msg(_("No Terminated Jobs.\n")); @@ -1092,6 +1324,8 @@ static void list_terminated_jobs(UAContext *ua) ua->send_msg(_("\nTerminated Jobs:\n")); ua->send_msg(_(" JobId Level Files Bytes Status Finished Name \n")); ua->send_msg(_("====================================================================\n")); + } else if (ua->api > 1) { + ua->send_msg(ow.start_group("terminated")); } foreach_dlist(je, last_jobs) { char JobName[MAX_NAME_LENGTH]; @@ -1151,13 +1385,31 @@ static void list_terminated_jobs(UAContext *ua) break; } if (ua->api == 1) { - ua->send_msg(_("%6d\t%-6s\t%8s\t%10s\t%-7s\t%-8s\t%s\n"), + ua->send_msg(_("%7d\t%-6s\t%8s\t%10s\t%-7s\t%-8s\t%s\n"), je->JobId, level, edit_uint64_with_commas(je->JobFiles, b1), edit_uint64_with_suffix(je->JobBytes, b2), termstat, dt, JobName); + } else if (ua->api > 1) { + ua->send_msg("%s", + ow.get_output(OT_CLEAR, + OT_START_OBJ, + OT_INT32, "jobid", je->JobId, + OT_JOBLEVEL,"level", je->JobLevel, + OT_JOBTYPE, "type", je->JobType, + OT_JOBSTATUS,"status", je->JobStatus, + OT_STRING, "status_desc",termstat, + OT_SIZE, "jobbytes", je->JobBytes, + OT_INT32, "jobfiles", je->JobFiles, + OT_STRING, "job", je->Job, + OT_UTIME, "starttime", je->start_time, + OT_UTIME, "endtime", je->end_time, + OT_INT32, "errors", je->Errors, + OT_END_OBJ, + OT_END)); + } else { ua->send_msg(_("%6d %-7s %8s %10s %-7s %-8s %s\n"), je->JobId, @@ -1170,6 +1422,8 @@ static void list_terminated_jobs(UAContext *ua) } if (!ua->api) { ua->send_msg(_("\n")); + } else if (ua->api > 1) { + ua->send_msg(ow.end_group(false)); } unlock_last_jobs_list(); } diff --git a/bacula/src/dird/ua_tree.c b/bacula/src/dird/ua_tree.c index d6fb0f715e..24ad9cf186 100644 --- a/bacula/src/dird/ua_tree.c +++ b/bacula/src/dird/ua_tree.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,19 +11,17 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- User Agent Database File tree for Restore * command. This file interacts with the user implementing the * UA tree commands. * * Kern Sibbald, July MMII - * */ #include "bacula.h" @@ -198,6 +196,7 @@ int insert_tree_handler(void *ctx, int num_fields, char **row) type = TN_FILE; } decode_stat(row[4], &statp, sizeof(statp), &LinkFI); + hard_link = (LinkFI != 0); node = insert_tree_node(row[0], row[1], type, tree->root, NULL); JobId = str_to_int64(row[3]); @@ -251,7 +250,7 @@ int insert_tree_handler(void *ctx, int num_fields, char **row) node->type = type; node->soft_link = S_ISLNK(statp.st_mode) != 0; node->delta_seq = delta_seq; - + node->can_access = true; if (tree->all) { node->extract = true; /* extract all by default */ if (type == TN_DIR || type == TN_DIR_NLS) { @@ -831,11 +830,14 @@ static int cdcmd(UAContext *ua, TREE_CTX *tree) } if (!node) { ua->warning_msg(_("Invalid path given.\n")); - } else { + } + } + if (node) { + if (node->can_access) { tree->node = node; + } else { + ua->warning_msg(_("Invalid path given. Permission denied.\n")); } - } else { - tree->node = node; } return pwdcmd(ua, tree); } diff --git a/bacula/src/dird/ua_update.c b/bacula/src/dird/ua_update.c index 3ed16130e5..bc8fabb98f 100644 --- a/bacula/src/dird/ua_update.c +++ b/bacula/src/dird/ua_update.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -175,6 +175,24 @@ static void update_volretention(UAContext *ua, char *val, MEDIA_DBR *mr) } } +static void update_vol_cacheretention(UAContext *ua, char *val, MEDIA_DBR *mr) +{ + char ed1[150], ed2[50]; + POOL_MEM query(PM_MESSAGE); + if (!duration_to_utime(val, &mr->CacheRetention)) { + ua->error_msg(_("Invalid cache retention period specified: %s\n"), val); + return; + } + Mmsg(query, "UPDATE Media SET CacheRetention=%s WHERE MediaId=%s", + edit_uint64(mr->CacheRetention, ed1), edit_int64(mr->MediaId,ed2)); + if (!db_sql_query(ua->db, query.c_str(), NULL, NULL)) { + ua->error_msg("%s", db_strerror(ua->db)); + } else { + ua->info_msg(_("New Cache Retention period is: %s\n"), + edit_utime(mr->CacheRetention, ed1, sizeof(ed1))); + } +} + static void update_voluseduration(UAContext *ua, char *val, MEDIA_DBR *mr) { char ed1[150], ed2[50]; @@ -291,8 +309,9 @@ static void update_volslot(UAContext *ua, char *val, MEDIA_DBR *mr) return; } mr->Slot = atoi(val); - if (mr->Slot < 0) { - ua->error_msg(_("Invalid slot, it must be greater than zero\n")); + if (pr.MaxVols > 0 && mr->Slot > (int)pr.MaxVols) { + ua->error_msg(_("Invalid slot, it must be between 0 and MaxVols=%d\n"), + pr.MaxVols); return; } /* @@ -526,6 +545,7 @@ static int update_volume(UAContext *ua) NT_("RecyclePool"), /* 13 */ NT_("ActionOnPurge"), /* 14 */ NT_("FromAllPools"), /* 15 !!! see bellow !!! */ + NT_("CacheRetention"), /* 16 */ NULL }; #define FromPool 10 /* keep this updated */ @@ -602,6 +622,9 @@ static int update_volume(UAContext *ua) case 15: update_all_vols(ua); break; + case 16: + update_vol_cacheretention(ua, ua->argv[j], &mr); + break; } done = true; } @@ -626,12 +649,13 @@ static int update_volume(UAContext *ua) add_prompt(ua, _("Enabled")), /* 14 */ add_prompt(ua, _("RecyclePool")), /* 15 */ add_prompt(ua, _("Action On Purge")), /* 16 */ - add_prompt(ua, _("Done")); /* 17 */ + add_prompt(ua, _("Cache Retention")), /* 17 */ + add_prompt(ua, _("Done")); /* 18 */ i = do_prompt(ua, "", _("Select parameter to modify"), NULL, 0); /* For All Volumes, All Volumes from Pool, and Done, we don't need * a Volume record */ - if ( i != 12 && i != 13 && i != 17) { + if ( i != 12 && i != 13 && i != 18) { if (!select_media_dbr(ua, &mr)) { /* Get Volume record */ return 0; } @@ -825,6 +849,16 @@ static int update_volume(UAContext *ua) update_vol_actiononpurge(ua, ua->cmd, &mr); break; + case 17: + pm_strcpy(ret, ""); + ua->info_msg(_("Current Cache Retention period is: %s\n"), + edit_utime(mr.CacheRetention, ed1, sizeof(ed1))); + if (!get_cmd(ua, _("Enter Cache Retention period: "))) { + return 0; + } + update_vol_cacheretention(ua, ua->cmd, &mr); + break; + default: /* Done or error */ ua->info_msg(_("Selection terminated.\n")); return 1; @@ -896,17 +930,19 @@ static bool update_pool(UAContext *ua) */ static bool update_job(UAContext *ua) { - int i; - char ed1[50], ed2[50]; + int i, priority=0; + char ed1[50], ed2[50], ed3[50]; POOL_MEM cmd(PM_MESSAGE); JOB_DBR jr; CLIENT_DBR cr; + JCR *jcr; utime_t StartTime; char *client_name = NULL; char *start_time = NULL; const char *kw[] = { NT_("starttime"), /* 0 */ NT_("client"), /* 1 */ + NT_("priority"), /* 2 */ NULL }; Dmsg1(200, "cmd=%s\n", ua->cmd); @@ -918,11 +954,18 @@ static bool update_job(UAContext *ua) memset(&jr, 0, sizeof(jr)); memset(&cr, 0, sizeof(cr)); jr.JobId = str_to_int64(ua->argv[i]); + if (jr.JobId == 0) { + ua->error_msg("Bad jobid\n"); + return false; + } if (!db_get_job_record(ua->jcr, ua->db, &jr)) { ua->error_msg("%s", db_strerror(ua->db)); return false; } - + if (!acl_access_ok(ua, Job_ACL, jr.Name)) { + ua->error_msg(_("Update failed. Job not authorized on this console\n")); + return false; + } for (i=0; kw[i]; i++) { int j; if ((j=find_arg_with_value(ua, kw[i])) >= 0) { @@ -933,15 +976,32 @@ static bool update_job(UAContext *ua) case 1: /* Client name */ client_name = ua->argv[j]; break; + case 2: + priority = str_to_int64(ua->argv[j]); + break; } } } - if (!client_name && !start_time) { - ua->error_msg(_("Neither Client nor StartTime specified.\n")); + if (!client_name && !start_time && !priority) { + ua->error_msg(_("Neither Client, StartTime or Priority specified.\n")); return 0; } + if (priority > 0) { + foreach_jcr(jcr) { + if (jcr->JobId == jr.JobId) { + int old = jcr->JobPriority; + jcr->JobPriority = priority; + free_jcr(jcr); + ua->send_msg(_("Priority updated for running job \"%s\" from %d to %d\n"), jr.Job, old, priority); + return true; + } + } + endeach_jcr(jcr); + ua->error_msg(_("Job not found.\n")); + return true; + } if (client_name) { - if (!get_client_dbr(ua, &cr)) { + if (!get_client_dbr(ua, &cr, JT_BACKUP_RESTORE)) { return false; } jr.ClientId = cr.ClientId; @@ -972,8 +1032,8 @@ static bool update_job(UAContext *ua) jr.cStartTime, jr.cSchedTime, jr.cEndTime, - edit_uint64(jr.JobTDate, ed1), - edit_int64(jr.JobId, ed2)); + edit_uint64(jr.JobTDate, ed2), + edit_int64(jr.JobId, ed3)); if (!db_sql_query(ua->db, cmd.c_str(), NULL, NULL)) { ua->error_msg("%s", db_strerror(ua->db)); return false; diff --git a/bacula/src/dird/vbackup.c b/bacula/src/dird/vbackup.c index 09ab291a85..7cd276a40d 100644 --- a/bacula/src/dird/vbackup.c +++ b/bacula/src/dird/vbackup.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -17,21 +17,18 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- vbackup.c -- responsible for doing virtual * backup jobs or in other words, consolidation or synthetic - * backups. + * backups. No connection to the Client is made. * * Kern Sibbald, July MMVIII * * Basic tasks done here: - * Open DB and create records for this job. - * Figure out what Jobs to copy. - * Open Message Channel with Storage daemon to tell him a job will be starting. - * Open connection with File daemon and pass him commands - * to do the backup. - * When the File daemon finishes the job, update the DB. - * + * Open DB and create records for this job. + * Figure out what Jobs to copy. + * Open Message Channel with Storage daemon to tell him a + * job will be starting. + * Connect to the storage daemon and run the job. */ #include "bacula.h" @@ -49,14 +46,30 @@ void vbackup_cleanup(JCR *jcr, int TermCode); */ bool do_vbackup_init(JCR *jcr) { + if (!get_or_create_fileset_record(jcr)) { + Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId); + return false; + } - /* - * if the read pool has not been allocated yet due to the job - * being upgraded to a virtual full then allocate it now - */ - if (!jcr->rpool_source) - jcr->rpool_source = get_pool_memory(PM_MESSAGE); + apply_pool_overrides(jcr); + + if (!allow_duplicate_job(jcr)) { + return false; + } + /* + * If the read pool has not been allocated yet due to the job + * being upgraded to a virtual full then allocate it now + */ + if (!jcr->rpool_source) { + jcr->rpool_source = get_pool_memory(PM_MESSAGE); + } + jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name()); + if (jcr->jr.PoolId == 0) { + Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId); + Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n")); + return false; + } /* * Note, at this point, pool is the pool for this job. We * transfer it to rpool (read pool), and a bit later, @@ -102,6 +115,7 @@ bool do_vbackup(JCR *jcr) char *p; sellist sel; db_list_ctx jobids; + UAContext *ua; Dmsg2(100, "rstorage=%p wstorage=%p\n", jcr->rstorage, jcr->wstorage); Dmsg2(100, "Read store=%s, write store=%s\n", @@ -205,10 +219,31 @@ _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n") Jmsg(jcr, M_FATAL, 0, _("No previous Jobs found.\n")); return false; } + jobids.count -= jcr->job->BackupsToKeep; + if (jobids.count <= 0) { + Jmsg(jcr, M_WARNING, 0, _("Insufficient Backups to Keep.\n")); + return false; + } + if (jobids.count == 1) { + Jmsg(jcr, M_WARNING, 0, _("Only one Job found. Consolidation not needed.\n")); + return false; + } + + /* Remove number of JobIds we want to keep */ + for (int i=0; i < (int)jcr->job->BackupsToKeep; i++) { + p = strrchr(jobids.list, ','); /* find last jobid */ + if (p == NULL) { + break; + } else { + *p = 0; + } + } /* Full by default, or might be Incr/Diff when jobid= is used */ jcr->jr.JobLevel = level_computed; + Jmsg(jcr, M_INFO, 0, "Consolidating JobIds=%s\n", jobids.list); + /* * Now we find the last job that ran and store it's info in * the previous_jr record. We will set our times to the @@ -274,10 +309,6 @@ _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n") jcr->jr.JobTDate = jcr->start_time; jcr->setJobStatus(JS_Running); - /* Add the following when support for base jobs is added to virtual full */ - //jcr->HasBase = jcr->job->base != NULL; - //jcr->jr.HasBase = jcr->HasBase; - /* Update job start record */ if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); @@ -313,6 +344,12 @@ _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n") if (jcr->JobStatus != JS_Terminated) { return false; } + if (jcr->job->DeleteConsolidatedJobs) { + ua = new_ua_context(jcr); + purge_jobs_from_catalog(ua, jobids.list); + free_ua_context(ua); + Jmsg(jcr, M_INFO, 0, _("Deleted consolidated JobIds=%s\n"), jobids.list); + } vbackup_cleanup(jcr, jcr->JobStatus); return true; @@ -336,7 +373,7 @@ void vbackup_cleanup(JCR *jcr, int TermCode) utime_t RunTime; POOL_MEM query(PM_MESSAGE); - Dmsg2(100, "Enter backup_cleanup %d %c\n", TermCode, TermCode); + Dmsg2(100, "Enter vbackup_cleanup %d %c\n", TermCode, TermCode); memset(&cr, 0, sizeof(cr)); jcr->setJobLevel(L_FULL); /* we want this to appear as a Full backup */ @@ -413,10 +450,9 @@ void vbackup_cleanup(JCR *jcr, int TermCode) bstrftimes(edt, sizeof(edt), jcr->jr.EndTime); RunTime = jcr->jr.EndTime - jcr->jr.StartTime; if (RunTime <= 0) { - kbps = 0; - } else { - kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime); + RunTime = 1; } + kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime); if (!db_get_job_volume_names(jcr, jcr->db, jcr->jr.JobId, &jcr->VolumeName)) { /* * Note, if the job has erred, most likely it did not write any @@ -512,11 +548,11 @@ int insert_bootstrap_handler(void *ctx, int num_fields, char **row) { JobId_t JobId; int FileIndex; - RBSR *bsr = (RBSR *)ctx; + rblist *bsr_list = (rblist *)ctx; JobId = str_to_int64(row[3]); FileIndex = str_to_int64(row[2]); - add_findex(bsr, JobId, FileIndex); + add_findex(bsr_list, JobId, FileIndex); return 0; } @@ -525,9 +561,10 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids) { RESTORE_CTX rx; UAContext *ua; + RBSR *bsr = NULL; memset(&rx, 0, sizeof(rx)); - rx.bsr = new_bsr(); + rx.bsr_list = New(rblist(bsr, &bsr->link)); ua = new_ua_context(jcr); rx.JobIds = jobids; @@ -540,7 +577,7 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids) if (!db_get_file_list(jcr, jcr->db_batch, jobids, false /* don't use md5 */, true /* use delta */, - insert_bootstrap_handler, (void *)rx.bsr)) + insert_bootstrap_handler, (void *)rx.bsr_list)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db_batch)); } @@ -560,7 +597,7 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids) */ Mmsg(rx.query, uar_sel_files, edit_int64(JobId, ed1)); Dmsg1(100, "uar_sel_files=%s\n", rx.query); - if (!db_sql_query(ua->db, rx.query, insert_bootstrap_handler, (void *)rx.bsr)) { + if (!db_sql_query(ua->db, rx.query, insert_bootstrap_handler, (void *)rx.bsr_list)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(ua->db)); } free_pool_memory(rx.query); @@ -568,11 +605,14 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids) } #endif - complete_bsr(ua, rx.bsr); + complete_bsr(ua, rx.bsr_list); jcr->ExpectedFiles = write_bsr_file(ua, rx); + if (chk_dbglvl(10)) { + Pmsg1(000, "Found %d files to consolidate.\n", jcr->ExpectedFiles); + } Jmsg(jcr, M_INFO, 0, _("Found %d files to consolidate into Virtual Full.\n"), - jcr->ExpectedFiles); + jcr->ExpectedFiles); free_ua_context(ua); - free_bsr(rx.bsr); + free_bsr(rx.bsr_list); return jcr->ExpectedFiles==0?false:true; } diff --git a/bacula/src/dird/verify.c b/bacula/src/dird/verify.c index b9f1c4f84e..76031dd4ad 100644 --- a/bacula/src/dird/verify.c +++ b/bacula/src/dird/verify.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -17,7 +17,6 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director -- verify.c -- responsible for running file verification * * Kern Sibbald, October MM @@ -28,7 +27,6 @@ * to do the verify. * When the File daemon sends the attributes, compare them to * what is in the DB. - * */ @@ -310,6 +308,8 @@ bool do_verify(JCR *jcr) level = "catalog"; break; case L_VERIFY_DATA: + send_accurate_current_files(jcr); + /* Fall-through wanted */ case L_VERIFY_VOLUME_TO_CATALOG: if (jcr->sd_calls_client) { if (jcr->FDVersion < 10) { @@ -500,6 +500,10 @@ void verify_cleanup(JCR *jcr, int TermCode) jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg)); if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || jcr->getJobLevel() == L_VERIFY_DATA) { + const char *accurate = "yes"; + if (jcr->is_JobLevel(L_VERIFY_DATA)) { + accurate = jcr->accurate ? "yes": "no"; + } jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n" " Build OS: %s %s %s\n" @@ -512,6 +516,7 @@ void verify_cleanup(JCR *jcr, int TermCode) " Verify Job: %s\n" " Start time: %s\n" " End time: %s\n" +" Accurate: %s\n" " Files Expected: %s\n" " Files Examined: %s\n" " Non-fatal FD errors: %d\n" @@ -530,6 +535,7 @@ void verify_cleanup(JCR *jcr, int TermCode) Name, sdt, edt, + accurate, edit_uint64_with_commas(jcr->ExpectedFiles, ec1), edit_uint64_with_commas(jcr->JobFiles, ec2), jcr->JobErrors, @@ -823,11 +829,11 @@ void get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId) */ jcr->fn_printed = false; bsnprintf(buf, sizeof(buf), - "SELECT Path.Path,Filename.Name FROM File,Path,Filename " - "WHERE File.JobId=%d AND File.FileIndex > 0 " - "AND File.MarkId!=%d AND File.PathId=Path.PathId " - "AND File.FilenameId=Filename.FilenameId", - JobId, jcr->JobId); + "SELECT Path.Path,Filename.Name FROM File,Path,Filename " + "WHERE File.JobId=%d AND File.FileIndex > 0 " + "AND File.MarkId!=%d AND File.PathId=Path.PathId " + "AND File.FilenameId=Filename.FilenameId", + JobId, jcr->JobId); /* missing_handler is called for each file found */ db_sql_query(jcr->db, buf, missing_handler, (void *)jcr); if (jcr->fn_printed) { diff --git a/bacula/src/filed/Makefile.in b/bacula/src/filed/Makefile.in index 5a6e67d1f9..8b3e7f2d4a 100644 --- a/bacula/src/filed/Makefile.in +++ b/bacula/src/filed/Makefile.in @@ -38,6 +38,8 @@ SVRSRCS = filed.c authenticate.c backup.c crypto.c \ xacl.c xacl_linux.c xacl_osx.c xacl_solaris.c xacl_freebsd.c SVROBJS = $(SVRSRCS:.c=.o) +JSONOBJS = bfdjson.o filed_conf.o + # these are the objects that are changed by the .configure process EXTRAOBJS = @OBJLIST@ @@ -56,7 +58,7 @@ LZO_INC= @LZO_INC@ @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< #------------------------------------------------------------------------- -all: Makefile bacula-fd @STATIC_FD@ +all: Makefile bacula-fd @STATIC_FD@ bfdjson @echo "==== Make of filed is good ====" @echo " " @@ -70,6 +72,12 @@ bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../ $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \ $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) +bfdjson: Makefile $(JSONOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@ + @echo "Linking $@ ..." + $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(JSONOBJS) \ + $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \ + $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) + static-bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind.a ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -static -L../lib -L../findlib -o $@ $(SVROBJS) \ $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \ @@ -100,6 +108,7 @@ devclean: realclean install: all $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-fd $(DESTDIR)$(sbindir)/bacula-fd + $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bfdjson $(DESTDIR)$(sbindir)/bfdjson @srcconf=bacula-fd.conf; \ if test -f ${DESTDIR}${sysconfdir}/$$srcconf; then \ destconf=$$srcconf.new; \ @@ -117,7 +126,7 @@ install: all fi uninstall: - (cd $(DESTDIR)$(sysconfdir); $(RMF) bacula-fd.conf) + (cd $(DESTDIR)$(sysconfdir); $(RMF) bacula-fd.conf bfdjson) (cd $(DESTDIR)$(sysconfdir); $(RMF) bacula-fd.conf.new) # Semi-automatic generation of dependencies: diff --git a/bacula/src/filed/accurate.c b/bacula/src/filed/accurate.c index f996b347f2..94911ef571 100644 --- a/bacula/src/filed/accurate.c +++ b/bacula/src/filed/accurate.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -164,6 +164,37 @@ static bool accurate_send_deleted_list(JCR *jcr) return true; } + +/* This function is called at the end of verify job + * We walk over all hash disk element, and we check + * for elt.seen. + */ +static bool accurate_check_deleted_list(JCR *jcr) +{ + bool ret=true; + CurFile *elt; + + if (!jcr->accurate) { + return true; + } + + if (jcr->file_list == NULL) { + return true; + } + + foreach_htable(elt, jcr->file_list) { + if (elt->seen) { + continue; + } + if (ret) { + Jmsg(jcr, M_INFO, 0, _("The following files were in the Catalog, but not in the Job data:\n"), elt->fname); + } + ret = false; + Jmsg(jcr, M_INFO, 0, _(" %s\n"), elt->fname); + } + return ret; +} + void accurate_free(JCR *jcr) { if (jcr->file_list) { @@ -187,6 +218,9 @@ bool accurate_finish(JCR *jcr) if (!jcr->rerunning) { ret = accurate_send_base_file_list(jcr); } + } else if (jcr->is_JobLevel(L_VERIFY_DATA)) { + ret = accurate_check_deleted_list(jcr); + } else { ret = accurate_send_deleted_list(jcr); } @@ -231,6 +265,60 @@ static bool accurate_add_file(JCR *jcr, uint32_t len, return ret; } +bool accurate_check_file(JCR *jcr, ATTR *attr, char *digest) +{ + struct stat statc; + int32_t LinkFIc; + bool stat = false; + char ed1[50], ed2[50]; + CurFile elt; + + if (!jcr->accurate) { + goto bail_out; + } + + if (!jcr->file_list) { + goto bail_out; /* Not initialized properly */ + } + + if (!accurate_lookup(jcr, attr->fname, &elt)) { + Dmsg1(dbglvl, "accurate %s (not found)\n", attr->fname); + stat = true; + goto bail_out; + } + decode_stat(elt.lstat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */ + + /* + * Loop over options supplied by user and verify the + * fields he requests. + */ + if (statc.st_size != attr->statp.st_size) { + Dmsg3(50, "%s st_size differs. Cat: %s File: %s\n", + attr->fname, + edit_uint64((uint64_t)statc.st_size, ed1), + edit_uint64((uint64_t)attr->statp.st_size, ed2)); + Jmsg(jcr, M_INFO, 0, "Cat st_size differs: %s\n", attr->fname); + stat = true; + } + + if (*elt.chksum && digest && *digest) { + if (strcmp(digest, elt.chksum)) { + Dmsg3(50, "%s chksum differs. Cat: %s File: %s\n", + attr->fname, + elt.chksum, + digest); + Jmsg(jcr, M_INFO, 0, "Cat checksum differs: %s\n", attr->fname); + stat = true; + } + } + + accurate_mark_file_as_seen(jcr, &elt); + +bail_out: + return stat; + +} + /* * This function is called for each file seen in fileset. * We check in file_list hash if fname have been backuped diff --git a/bacula/src/filed/authenticate.c b/bacula/src/filed/authenticate.c index fe2615d8df..81c8a0e126 100644 --- a/bacula/src/filed/authenticate.c +++ b/bacula/src/filed/authenticate.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -30,6 +30,22 @@ extern CLIENT *me; /* my resource */ const int dbglvl = 50; +/* Version at end of Hello + * prior to 10Mar08 no version + * 1 10Mar08 + * 2 13Mar09 - added the ability to restore from multiple storages + * 3 03Sep10 - added the restore object command for vss plugin 4.0 + * 4 25Nov10 - added bandwidth command 5.1 + * 5 24Nov11 - added new restore object command format (pluginname) 6.0 + * 6 15Feb12 - added Component selection information list + * 7 19Feb12 - added Expected files to restore + * 8 22Mar13 - added restore options + version for SD + * 9 06Aug13 - added comm line compression + * 10 01Jan14 - added SD Calls Client and api version to status command + */ +#define FD_VERSION 10 + +/* For compatibility with old Community SDs */ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; /* @@ -229,6 +245,13 @@ bool authenticate_storagedaemon(JCR *jcr) goto auth_fatal; } sscanf(sd->msg, "3000 OK Hello %d", &sd_version); + if (sd_version >= 1 && me->comm_compression) { + sd->set_compress(); + } else { + sd->clear_compress(); + Dmsg0(050, "*** No FD compression with SD\n"); + } + /* At this point, we have successfully connected */ auth_fatal: diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 5b2138e729..6e06cbd3d3 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -245,7 +245,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) jcr->last_stat_time = now; jcr->stat_interval = 30; /* Default 30 seconds */ } else if (now >= jcr->last_stat_time + jcr->stat_interval) { - jcr->dir_bsock->fsend("Progress Job=x files=%ld bytes=%lld bps=%ld\n", + jcr->dir_bsock->fsend("Progress JobId=x files=%ld bytes=%lld bps=%ld\n", jcr->JobFiles, jcr->JobBytes, jcr->LastRate); jcr->last_stat_time = now; } @@ -879,6 +879,16 @@ bool encode_and_send_attributes(bctx_t &bctx) pm_strcpy(jcr->last_fname, ff_pkt->fname); jcr->unlock(); + /* Display the information about the current file if requested */ + if (is_message_type_set(jcr, M_SAVED)) { + ATTR attr; + memcpy(&attr.statp, &ff_pkt->statp, sizeof(struct stat)); + attr.type = ff_pkt->type; + attr.ofname = (POOLMEM *)ff_pkt->fname; + attr.olname = (POOLMEM *)ff_pkt->link; + print_ls_output(jcr, &attr, M_SAVED); + } + /* Debug code: check if we must hangup */ if (hangup > 0 && (jcr->JobFiles > (uint32_t)hangup)) { jcr->setJobStatus(JS_Incomplete); diff --git a/bacula/src/filed/backup.h b/bacula/src/filed/backup.h index 31090e17f8..1fe96b6501 100644 --- a/bacula/src/filed/backup.h +++ b/bacula/src/filed/backup.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -49,12 +49,10 @@ struct bctx_t { /* Compression variables */ #if defined(HAVE_LIBZ) || defined(HAVE_LZO) - uLong max_compress_len; uLong compress_len; + uLong max_compress_len; Bytef *cbuf; Bytef *cbuf2; -#else - uint64_t max_compress_len; #endif #ifdef HAVE_LZO comp_stream_header ch; diff --git a/bacula/src/filed/bfdjson.c b/bacula/src/filed/bfdjson.c new file mode 100644 index 0000000000..da909483d8 --- /dev/null +++ b/bacula/src/filed/bfdjson.c @@ -0,0 +1,608 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2016 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula File Daemon to Json + * + * Kern Sibbald, Sept MMXII + * + */ + +#include "bacula.h" +#include "filed.h" + +/* Imported Functions */ +extern bool parse_fd_config(CONFIG *config, const char *configfile, int exit_code); +void store_msgs(LEX *lc, RES_ITEM *item, int index, int pass); + +typedef struct +{ + /* default { { "Director": { "Name": aa, ...} }, { "Job": {..} */ + bool do_list; /* [ {}, {}, ..] or { "aa": {}, "bb": {}, ...} */ + bool do_one; /* { "Name": "aa", "Description": "test, ... } */ + bool do_only_data; /* [ {}, {}, {}, ] */ + char *resource_type; + char *resource_name; + regex_t directive_reg; +} display_filter; + +/* Forward referenced functions */ +static bool check_resources(); +static void sendit(void *sock, const char *fmt, ...); +static void dump_json(display_filter *filter); + +/* Exported variables */ +CLIENT *me; /* my resource */ +#if defined(_MSC_VER) +extern "C" { // work around visual compiler mangling variables + extern URES res_all; +} +#else +extern URES res_all; +#endif +extern s_kw msg_types[]; +extern RES_TABLE resources[]; + +#define CONFIG_FILE "bacula-fd.conf" /* default config file */ + +char *configfile = NULL; +static CONFIG *config; + +static void usage() +{ + fprintf(stderr, _( +PROG_COPYRIGHT +"\n%sVersion: %s (%s)\n\n" +"Usage: bfdjson [options] [config_file]\n" +" -r get resource type \n" +" -n get resource \n" +" -l get only directives matching dirs (use with -r)\n" +" -D get only data\n" +" -c use as configuration file\n" +" -d set debug level to \n" +" -dt print a timestamp in debug output\n" +" -t test configuration file and exit\n" +" -v verbose user messages\n" +" -? print this message.\n" +"\n"), 2012, "", VERSION, BDATE); + + exit(1); +} + + +/********************************************************************* + * + * Bacula File daemon to Json + * + */ + +int main (int argc, char *argv[]) +{ + int ch; + bool test_config = false; + display_filter filter; + memset(&filter, 0, sizeof(filter)); + + setlocale(LC_ALL, ""); + bindtextdomain("bacula", LOCALEDIR); + textdomain("bacula"); + + my_name_is(argc, argv, "bacula-fd"); + init_msg(NULL, NULL); + + while ((ch = getopt(argc, argv, "Dr:n:c:d:tv?l:")) != -1) { + switch (ch) { + case 'D': + filter.do_only_data = true; + break; + case 'l': + /* Might use something like -l '^(Name|Description)$' */ + filter.do_list = true; + if (regcomp(&filter.directive_reg, optarg, REG_EXTENDED) != 0) { + Jmsg((JCR *)NULL, M_ERROR_TERM, 0, + _("Please use valid -l argument: %s\n"), optarg); + } + break; + + case 'r': + filter.resource_type = optarg; + break; + + case 'n': + filter.resource_name = optarg; + break; + + case 'c': /* configuration file */ + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(optarg); + break; + + case 'd': /* debug level */ + if (*optarg == 't') { + dbg_timestamp = true; + } else { + debug_level = atoi(optarg); + if (debug_level <= 0) { + debug_level = 1; + } + } + break; + + case 't': + test_config = true; + break; + + case 'v': /* verbose */ + verbose++; + break; + + case '?': + default: + usage(); + + } + } + argc -= optind; + argv += optind; + + if (argc) { + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(*argv); + argc--; + argv++; + } + if (argc) { + usage(); + } + + if (filter.do_list && !filter.resource_type) { + usage(); + } + + if (filter.resource_type && filter.resource_name) { + filter.do_one = true; + } + + if (configfile == NULL || configfile[0] == 0) { + configfile = bstrdup(CONFIG_FILE); + } + + if (test_config && verbose > 0) { + char buf[1024]; + find_config_file(configfile, buf, sizeof(buf)); + sendit(NULL, "config_file=%s\n", buf); + } + + config = New(CONFIG()); + config->encode_password(false); + parse_fd_config(config, configfile, M_ERROR_TERM); + + if (!check_resources()) { + Emsg1(M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); + terminate_filed(1); + } + + if (test_config) { + terminate_filed(0); + } + + dump_json(&filter); + + if (filter.do_list) { + regfree(&filter.directive_reg); + } + + terminate_filed(0); + exit(0); /* should never get here */ +} + +void terminate_filed(int sig) +{ + static bool already_here = false; + + if (already_here) { + bmicrosleep(2, 0); /* yield */ + exit(1); /* prevent loops */ + } + already_here = true; + debug_level = 0; /* turn off debug */ + + if (configfile != NULL) { + free(configfile); + } + + if (debug_level > 0) { + print_memory_pool_stats(); + } + if (config) { + delete config; + config = NULL; + } + term_msg(); + free(res_head); + res_head = NULL; + close_memory_pool(); /* release free memory in pool */ + //sm_dump(false); /* dump orphaned buffers */ + exit(sig); +} + +/* + * Dump out all resources in json format. + * Note!!!! This routine must be in this file rather + * than in src/lib/parser_conf.c otherwise the pointers + * will be all messed up. + */ +static void dump_json(display_filter *filter) +{ + int resinx, item, directives, first_directive; + bool first_res; + RES_ITEM *items; + RES *res; + HPKT hpkt; + regmatch_t pmatch[32]; + + init_hpkt(hpkt); + + me = (CLIENT *)GetNextRes(R_CLIENT, NULL); + + if (filter->do_only_data) { + sendit(NULL, "["); + + /* List resources and directives */ + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } + * or print a single item + */ + } else if (filter->do_one || filter->do_list) { + sendit(NULL, "{"); + + } else { + /* [ { "Client": { "Name": "aa",.. } }, { "Director": { "Name": "bb", ... } } ]*/ + sendit(NULL, "["); + } + + first_res = true; + + /* Loop over all resource types */ + for (resinx=0; resources[resinx].name; resinx++) { + /* Skip Client alias */ + if (strcmp(resources[resinx].name, "Client") == 0) { + continue; + } + + /* Skip this resource type */ + if (filter->resource_type && + strcasecmp(filter->resource_type, resources[resinx].name) != 0) { + continue; + } + + directives = 0; + /* Loop over all resources of this type */ + foreach_rblist(res, res_head[resinx]->res_list) { + hpkt.res = res; + items = resources[resinx].items; + if (!items) { + break; + } + /* Copy the resource into res_all */ + memcpy(&res_all, res, sizeof(res_all)); + + if (filter->resource_name) { + bool skip=true; + /* The Name should be at the first place, so this is not a real loop */ + for (item=0; items[item].name; item++) { + if (strcasecmp(items[item].name, "Name") == 0) { + if (strcasecmp(*(items[item].value), filter->resource_name) == 0) { + skip = false; + } + break; + } + } + if (skip) { /* The name doesn't match, so skip it */ + continue; + } + } + + if (!first_res) { + printf(",\n"); + } + + first_directive = 0; + + if (filter->do_only_data) { + sendit(NULL, " {"); + + } else if (filter->do_one) { + /* Nothing to print */ + + /* When sending the list, the form is: + * { aa: { Name: aa, Description: aadesc...}, bb: { Name: bb + */ + } else if (filter->do_list) { + /* Search and display Name, should be the first item */ + for (item=0; items[item].name; item++) { + if (strcmp(items[item].name, "Name") == 0) { + sendit(NULL, "%s: {\n", quote_string(hpkt.edbuf2, *items[item].value)); + break; + } + } + } else { + /* Begin new resource */ + sendit(NULL, "{\n \"%s\": {", resources[resinx].name); + } + + /* dirtry trick for a deprecated directive */ + bool dedup_index_directory_set = false; + + /* Loop over all items in the resource */ + for (item=0; items[item].name; item++) { + /* Check user argument -l */ + if (filter->do_list && + regexec(&filter->directive_reg, + items[item].name, 32, pmatch, 0) != 0) + { + continue; + } + + /* Special tweak for a deprecated variable */ + if (strcmp(items[item].name, "DedupIndexDirectory") == 0) { + dedup_index_directory_set = bit_is_set(item, res_all.hdr.item_present); + continue; + } + if (strcmp(items[item].name, "EnableClientRehydration") == 0) { + if (dedup_index_directory_set && !bit_is_set(item, res_all.hdr.item_present)) { + set_bit(item, res_all.hdr.item_present); + *(bool *)(items[item].value) = true; + } + } + + hpkt.ritem = &items[item]; + if (bit_is_set(item, res_all.hdr.item_present)) { + if (first_directive++ > 0) printf(","); + if (display_global_item(hpkt)) { + /* Fall-through wanted */ + } else { + printf("\n \"%s\": null", items[item].name); + } + directives++; + } else { /* end if is present */ + /* For some directives, the bit_is_set() is not set (e.g. addresses) */ + if (me && strcmp(resources[resinx].name, "FileDaemon") == 0) { + if (strcmp(items[item].name, "FdPort") == 0) { + if (get_first_port_host_order(me->FDaddrs) != items[item].default_value) { + if (first_directive++ > 0) printf(","); + printf("\n \"FdPort\": %d", + get_first_port_host_order(me->FDaddrs)); + } + } else if (me && strcmp(items[item].name, "FdAddress") == 0) { + char buf[500]; + get_first_address(me->FDaddrs, buf, sizeof(buf)); + if (strcmp(buf, "0.0.0.0") != 0) { + if (first_directive++ > 0) printf(","); + printf("\n \"FdAddress\": \"%s\"", buf); + } + } else if (me && strcmp(items[item].name, "FdSourceAddress") == 0 + && me->FDsrc_addr) { + char buf[500]; + get_first_address(me->FDsrc_addr, buf, sizeof(buf)); + if (strcmp(buf, "0.0.0.0") != 0) { + if (first_directive++ > 0) printf(","); + printf("\n \"FdSourceAddress\": \"%s\"", buf); + } + } + } + } + if (items[item].flags & ITEM_LAST) { + display_last(hpkt); /* If last bit set always call to cleanup */ + } + } + + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } */ + if (filter->do_only_data || filter->do_list) { + sendit(NULL, "\n }"); /* Finish the Resource with a single } */ + + } else { + if (filter->do_one) { + /* don't print anything */ + } else if (first_directive > 0) { + sendit(NULL, "\n }\n}"); /* end of resource */ + } else { + sendit(NULL, "}\n}"); + } + } + first_res = false; + } /* End loop over all resources of this type */ + } /* End loop all resource types */ + + if (filter->do_only_data) { + sendit(NULL, "\n]\n"); + + /* In list context, we are dealing with a hash */ + } else if (filter->do_one || filter->do_list) { + sendit(NULL, "\n}\n"); + + } else { + sendit(NULL, "\n]\n"); + } + term_hpkt(hpkt); +} + + +/* +* Make a quick check to see that we have all the +* resources needed. +*/ +static bool check_resources() +{ + bool OK = true; + DIRRES *director; + CONSRES *cons; + bool need_tls; + + LockRes(); + + me = (CLIENT *)GetNextRes(R_CLIENT, NULL); + if (!me) { + Emsg1(M_FATAL, 0, _("No File daemon resource defined in %s\n" + "Without that I don't know who I am :-(\n"), configfile); + OK = false; + } else { + if (GetNextRes(R_CLIENT, (RES *) me) != NULL) { + Emsg1(M_FATAL, 0, _("Only one Client resource permitted in %s\n"), + configfile); + OK = false; + } + my_name_is(0, NULL, me->hdr.name); + if (!me->messages) { + me->messages = (MSGS *)GetNextRes(R_MSGS, NULL); + if (!me->messages) { + Emsg1(M_FATAL, 0, _("No Messages resource defined in %s\n"), configfile); + OK = false; + } + } + /* tls_require implies tls_enable */ + if (me->tls_require) { +#ifndef HAVE_TLS + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; +#else + me->tls_enable = true; +#endif + } + need_tls = me->tls_enable || me->tls_authenticate; + + if ((!me->tls_ca_certfile && !me->tls_ca_certdir) && need_tls) { + Emsg1(M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for File daemon in %s.\n"), + configfile); + OK = false; + } + + /* pki_encrypt implies pki_sign */ + if (me->pki_encrypt) { + me->pki_sign = true; + } + + if ((me->pki_encrypt || me->pki_sign) && !me->pki_keypair_file) { + Emsg2(M_FATAL, 0, _("\"PKI Key Pair\" must be defined for File" + " daemon \"%s\" in %s if either \"PKI Sign\" or" + " \"PKI Encrypt\" are enabled.\n"), me->hdr.name, configfile); + OK = false; + } + } + + + /* Verify that a director record exists */ + LockRes(); + director = (DIRRES *)GetNextRes(R_DIRECTOR, NULL); + UnlockRes(); + if (!director) { + Emsg1(M_FATAL, 0, _("No Director resource defined in %s\n"), + configfile); + OK = false; + } + + foreach_res(director, R_DIRECTOR) { + /* tls_require implies tls_enable */ + if (director->tls_require) { +#ifndef HAVE_TLS + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; +#else + director->tls_enable = true; +#endif + } + need_tls = director->tls_enable || director->tls_authenticate; + + if (!director->tls_certfile && need_tls) { + Emsg2(M_FATAL, 0, _("\"TLS Certificate\" file not defined for Director \"%s\" in %s.\n"), + director->hdr.name, configfile); + OK = false; + } + + if (!director->tls_keyfile && need_tls) { + Emsg2(M_FATAL, 0, _("\"TLS Key\" file not defined for Director \"%s\" in %s.\n"), + director->hdr.name, configfile); + OK = false; + } + + if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && need_tls && director->tls_verify_peer) { + Emsg2(M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Director \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + director->hdr.name, configfile); + OK = false; + } + } + + foreach_res(cons, R_CONSOLE) { + /* tls_require implies tls_enable */ + if (cons->tls_require) { +#ifndef HAVE_TLS + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; +#else + cons->tls_enable = true; +#endif + } + need_tls = cons->tls_enable || cons->tls_authenticate; + + if (!cons->tls_certfile && need_tls) { + Emsg2(M_FATAL, 0, _("\"TLS Certificate\" file not defined for Console \"%s\" in %s.\n"), + cons->hdr.name, configfile); + OK = false; + } + + if (!cons->tls_keyfile && need_tls) { + Emsg2(M_FATAL, 0, _("\"TLS Key\" file not defined for Console \"%s\" in %s.\n"), + cons->hdr.name, configfile); + OK = false; + } + + if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) && need_tls && cons->tls_verify_peer) { + Emsg2(M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Console \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + cons->hdr.name, configfile); + OK = false; + } + } + + UnlockRes(); + + return OK; +} + +static void sendit(void *sock, const char *fmt, ...) +{ + char buf[3000]; + va_list arg_ptr; + + va_start(arg_ptr, fmt); + bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); + va_end(arg_ptr); + fputs(buf, stdout); + fflush(stdout); +} diff --git a/bacula/src/filed/fd_plugins.h b/bacula/src/filed/fd_plugins.h index f591572a29..5a2857ff79 100644 --- a/bacula/src/filed/fd_plugins.h +++ b/bacula/src/filed/fd_plugins.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Application Programming Interface (API) definition for Bacula Plugins * * Kern Sibbald, October 2007 - * */ #ifndef __FD_PLUGINS_H @@ -91,7 +90,7 @@ struct save_pkt { char *link; /* Link name if any */ struct stat statp; /* System stat() packet for file */ int32_t type; /* FT_xx for this file */ - uint32_t flags; /* Bacula internal flags */ + uint64_t flags; /* Bacula internal flags */ bool no_read; /* During the save, the file should not be saved */ bool portable; /* set if data format is portable */ bool accurate_found; /* Found in accurate list (valid after check_changes()) */ @@ -147,7 +146,7 @@ struct io_pkt { int32_t io_errno; /* errno code */ int32_t lerror; /* Win32 error code */ int32_t whence; /* lseek argument */ - boffset_t offset; /* lseek argument */ + boffset_t offset; /* lseek argument or in bread current offset*/ bool win32; /* Win32 GetLastError returned */ int32_t pkt_end; /* end packet sentinel */ }; diff --git a/bacula/src/filed/fd_snapshot.c b/bacula/src/filed/fd_snapshot.c index 135dca7c59..f92f2c0173 100644 --- a/bacula/src/filed/fd_snapshot.c +++ b/bacula/src/filed/fd_snapshot.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -272,16 +272,6 @@ void close_snapshot_backup_session(JCR *jcr) } } -/* Special cmd_parser subclass to not look after plugin - * names when decoding the line - */ -class arg_parser: public cmd_parser -{ -public: - arg_parser(): cmd_parser() { use_name = false; }; - virtual ~arg_parser() {}; -}; - class snapshot; /* Device that exists on the system */ diff --git a/bacula/src/filed/filed.c b/bacula/src/filed/filed.c index 677d172190..28b2baffd2 100644 --- a/bacula/src/filed/filed.c +++ b/bacula/src/filed/filed.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Bacula File Daemon * * Kern Sibbald, March MM - * */ #include "bacula.h" @@ -38,6 +37,9 @@ CLIENT *me; /* my resource */ bool no_signals = false; void *start_heap; extern struct s_cmds cmds[]; +extern dlist *daemon_msg_queue; +extern pthread_mutex_t daemon_msg_queue_mutex; + #ifndef CONFIG_FILE /* Might be overwritten */ #define CONFIG_FILE "bacula-fd.conf" /* default config file */ @@ -88,6 +90,7 @@ int main (int argc, char *argv[]) bool keep_readall_caps = false; char *uid = NULL; char *gid = NULL; + MQUEUE_ITEM *item = NULL; start_heap = sbrk(0); setlocale(LC_ALL, ""); @@ -98,6 +101,8 @@ int main (int argc, char *argv[]) my_name_is(argc, argv, PROG_NAME); init_msg(NULL, NULL); daemon_start_time = time(NULL); + /* Setup daemon message queue */ + daemon_msg_queue = New(dlist(item, &item->link)); while ((ch = getopt(argc, argv, "c:d:fg:kmstTu:v?D:")) != -1) { switch (ch) { @@ -188,23 +193,24 @@ int main (int argc, char *argv[]) } server_tid = pthread_self(); - if (!no_signals) { - init_signals(terminate_filed); - } else { - /* This reduces the number of signals facilitating debugging */ - watchdog_sleep_time = 120; /* long timeout for debugging */ - } if (configfile == NULL) { configfile = bstrdup(CONFIG_FILE); } - if (!foreground) { + if (!foreground && !test_config) { daemon_start(); init_stack_dump(); /* set new pid */ } - config = new_config_parser(); + if (!no_signals) { + init_signals(terminate_filed); + } else { + /* This reduces the number of signals facilitating debugging */ + watchdog_sleep_time = 120; /* long timeout for debugging */ + } + + config = New(CONFIG()); parse_fd_config(config, configfile, M_ERROR_TERM); if (init_crypto() != 0) { @@ -279,6 +285,11 @@ void terminate_filed(int sig) generate_daemon_event(NULL, "Exit"); unload_plugins(); + P(daemon_msg_queue_mutex); + daemon_msg_queue->destroy(); + free(daemon_msg_queue); + V(daemon_msg_queue_mutex); + if (!test_config) { write_state_file(me->working_directory, "bacula-fd", get_first_port_host_order(me->FDaddrs)); @@ -295,12 +306,13 @@ void terminate_filed(int sig) } if (config) { - config->free_resources(); - free(config); + delete config; config = NULL; } term_msg(); cleanup_crypto(); + free(res_head); + res_head = NULL; close_memory_pool(); /* release free memory in pool */ lmgr_cleanup_main(); sm_dump(false); /* dump orphaned buffers */ @@ -623,6 +635,60 @@ static bool check_resources() } } + CONSRES *console; + foreach_res(console, R_CONSOLE) { + /* tls_require implies tls_enable */ + if (console->tls_require) { +#ifndef HAVE_TLS + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; +#else + console->tls_enable = true; +#endif + } + need_tls = console->tls_enable || console->tls_authenticate; + + if (!console->tls_certfile && need_tls) { + Emsg2(M_FATAL, 0, _("\"TLS Certificate\" file not defined for Console \"%s\" in %s.\n"), + console->hdr.name, configfile); + OK = false; + } + + if (!console->tls_keyfile && need_tls) { + Emsg2(M_FATAL, 0, _("\"TLS Key\" file not defined for Console \"%s\" in %s.\n"), + console->hdr.name, configfile); + OK = false; + } + + if ((!console->tls_ca_certfile && !console->tls_ca_certdir) && need_tls && console->tls_verify_peer) { + Emsg2(M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Console \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + console->hdr.name, configfile); + OK = false; + } + + /* If everything is well, attempt to initialize our per-resource TLS context */ + if (OK && (need_tls || console->tls_require)) { + /* Initialize TLS context: + * Args: CA certfile, CA certdir, Certfile, Keyfile, + * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */ + console->tls_ctx = new_tls_context(console->tls_ca_certfile, + console->tls_ca_certdir, console->tls_certfile, + console->tls_keyfile, NULL, NULL, console->tls_dhfile, + console->tls_verify_peer); + + if (!console->tls_ctx) { + Emsg2(M_FATAL, 0, _("Failed to initialize TLS context for Console \"%s\" in %s.\n"), + console->hdr.name, configfile); + OK = false; + } + } + + } + UnlockRes(); if (OK) { diff --git a/bacula/src/filed/filed.h b/bacula/src/filed/filed.h index 735021e16e..0ef70ac6bc 100644 --- a/bacula/src/filed/filed.h +++ b/bacula/src/filed/filed.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -20,11 +20,10 @@ * Bacula File Daemon specific configuration and defines * * Kern Sibbald, Jan MMI - * */ //#define TEST_WORKER -#ifdef TEST_WORKER +#ifdef TEST_WORKER #define ERROR_BUFFER_OVERFLOW 1 #define ERROR_SUCCESS 0 #endif @@ -43,11 +42,11 @@ #include "findlib/find.h" #include "xacl.h" #include "jcr.h" -#include "protos.h" /* file daemon prototypes */ +#include "protos.h" /* file daemon prototypes */ #include "lib/runscript.h" #include "lib/breg.h" #ifdef HAVE_LIBZ -#include /* compression headers */ +#include /* compression headers */ #else #define uLongf uint32_t #endif @@ -56,8 +55,8 @@ #include #endif -extern CLIENT *me; /* "Global" Client resource */ -extern bool win32decomp; /* Use decomposition of BackupRead data */ +extern CLIENT *me; /* "Global" Client resource */ +extern bool win32decomp; /* Use decomposition of BackupRead data */ extern bool no_win32_write_errors; /* Ignore certain errors */ void terminate_filed(int sig); @@ -65,7 +64,7 @@ void terminate_filed(int sig); struct s_cmds { const char *cmd; int (*func)(JCR *); - int monitoraccess; /* specify if monitors have access to this function */ + int access; /* specify if monitors/restricted have access to this function */ }; void allow_os_suspensions(); diff --git a/bacula/src/filed/filed_conf.c b/bacula/src/filed/filed_conf.c index a091bbaba0..9bda54d8f8 100644 --- a/bacula/src/filed/filed_conf.c +++ b/bacula/src/filed/filed_conf.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -36,7 +36,6 @@ * for the resource records. * * Kern Sibbald, September MM - * */ #include "bacula.h" @@ -48,8 +47,7 @@ */ int32_t r_first = R_FIRST; int32_t r_last = R_LAST; -static RES *sres_head[R_LAST - R_FIRST + 1]; -RES **res_head = sres_head; +RES_HEAD **res_head; /* Forward referenced subroutines */ @@ -116,6 +114,7 @@ static RES_ITEM cli_items[] = { {"TlsKey", store_dir, ITEM(res_client.tls_keyfile), 0, 0, 0}, {"VerId", store_str, ITEM(res_client.verid), 0, 0, 0}, {"MaximumBandwidthPerJob",store_speed, ITEM(res_client.max_bandwidth_per_job), 0, 0, 0}, + {"CommCompression", store_bool, ITEM(res_client.comm_compression), 0, ITEM_DEFAULT, true}, {"DisableCommand", store_alist_str, ITEM(res_client.disable_cmds), 0, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -127,6 +126,7 @@ static RES_ITEM dir_items[] = { {"Password", store_password, ITEM(res_dir.password), 0, ITEM_REQUIRED, 0}, {"Address", store_str, ITEM(res_dir.address), 0, 0, 0}, {"Monitor", store_bool, ITEM(res_dir.monitor), 0, ITEM_DEFAULT, 0}, + {"Remote", store_bool, ITEM(res_dir.remote), 0, ITEM_DEFAULT, 0}, {"TlsAuthenticate", store_bool, ITEM(res_dir.tls_authenticate), 0, 0, 0}, {"TlsEnable", store_bool, ITEM(res_dir.tls_enable), 0, 0, 0}, {"TlsRequire", store_bool, ITEM(res_dir.tls_require), 0, 0, 0}, @@ -139,6 +139,27 @@ static RES_ITEM dir_items[] = { {"TlsAllowedCn", store_alist_str, ITEM(res_dir.tls_allowed_cns), 0, 0, 0}, {"MaximumBandwidthPerJob", store_speed, ITEM(res_dir.max_bandwidth_per_job), 0, 0, 0}, {"DisableCommand", store_alist_str, ITEM(res_dir.disable_cmds), 0, 0, 0}, + {"Console", store_res, ITEM(res_dir.console), R_CONSOLE, 0, 0}, + {NULL, NULL, {0}, 0, 0, 0} +}; + +/* Consoles that we can use to connect a Director */ +static RES_ITEM cons_items[] = { + {"Name", store_name, ITEM(res_cons.hdr.name), 0, ITEM_REQUIRED, 0}, + {"Description", store_str, ITEM(res_cons.hdr.desc), 0, 0, 0}, + {"Password", store_password, ITEM(res_cons.password), 0, ITEM_REQUIRED, 0}, + {"Address", store_str, ITEM(res_cons.address), 0, 0, 0}, + {"DirPort", store_pint32, ITEM(res_cons.DIRport), 0, ITEM_DEFAULT, 9101}, + {"TlsAuthenticate", store_bool, ITEM(res_cons.tls_authenticate), 0, 0, 0}, + {"TlsEnable", store_bool, ITEM(res_cons.tls_enable), 0, 0, 0}, + {"TlsRequire", store_bool, ITEM(res_cons.tls_require), 0, 0, 0}, + {"TlsVerifyPeer", store_bool, ITEM(res_cons.tls_verify_peer), 0, ITEM_DEFAULT, 1}, + {"TlsCaCertificateFile", store_dir, ITEM(res_cons.tls_ca_certfile), 0, 0, 0}, + {"TlsCaCertificateDir", store_dir, ITEM(res_cons.tls_ca_certdir), 0, 0, 0}, + {"TlsCertificate", store_dir, ITEM(res_cons.tls_certfile), 0, 0, 0}, + {"TlsKey", store_dir, ITEM(res_cons.tls_keyfile), 0, 0, 0}, + {"TlsDhFile", store_dir, ITEM(res_cons.tls_dhfile), 0, 0, 0}, + {"TlsAllowedCn", store_alist_str, ITEM(res_cons.tls_allowed_cns), 0, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -153,6 +174,7 @@ RES_TABLE resources[] = { {"Director", dir_items, R_DIRECTOR}, {"FileDaemon", cli_items, R_CLIENT}, {"Messages", msgs_items, R_MSGS}, + {"Console", cons_items, R_CONSOLE}, {"Client", cli_items, R_CLIENT}, /* alias for filedaemon */ {NULL, NULL, 0} }; @@ -242,6 +264,10 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, recurse = 0; } switch (type) { + case R_CONSOLE: + sendit(sock, "Console: name=%s password=%s\n", ares->name, + res->res_cons.password); + break; case R_DIRECTOR: sendit(sock, "Director: name=%s password=%s\n", ares->name, res->res_dir.password); @@ -261,8 +287,8 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, sendit(sock, "Unknown resource type %d\n", type); } ares = GetNextRes(type, ares); - if (recurse && res->res_dir.hdr.next) { - dump_resource(type, res->res_dir.hdr.next, sendit, sock); + if (recurse && ares) { + dump_resource(type, ares, sendit, sock); } } @@ -276,7 +302,6 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, */ void free_resource(RES *sres, int type) { - RES *nres; URES *res = (URES *)sres; if (res == NULL) { @@ -284,7 +309,6 @@ void free_resource(RES *sres, int type) } /* common stuff -- free the resource name */ - nres = (RES *)res->res_dir.hdr.next; if (res->res_dir.hdr.name) { free(res->res_dir.hdr.name); } @@ -327,7 +351,36 @@ void free_resource(RES *sres, int type) free(res->res_dir.disabled_cmds_array); } break; - case R_CLIENT: + case R_CONSOLE: + if (res->res_cons.password) { + free(res->res_cons.password); + } + if (res->res_cons.address) { + free(res->res_cons.address); + } + if (res->res_cons.tls_ctx) { + free_tls_context(res->res_cons.tls_ctx); + } + if (res->res_cons.tls_ca_certfile) { + free(res->res_cons.tls_ca_certfile); + } + if (res->res_cons.tls_ca_certdir) { + free(res->res_cons.tls_ca_certdir); + } + if (res->res_cons.tls_certfile) { + free(res->res_cons.tls_certfile); + } + if (res->res_cons.tls_keyfile) { + free(res->res_cons.tls_keyfile); + } + if (res->res_cons.tls_dhfile) { + free(res->res_cons.tls_dhfile); + } + if (res->res_cons.tls_allowed_cns) { + delete res->res_cons.tls_allowed_cns; + } + break; + case R_CLIENT: if (res->res_client.working_directory) { free(res->res_client.working_directory); } @@ -424,18 +477,16 @@ void free_resource(RES *sres, int type) if (res) { free(res); } - if (nres) { - free_resource(nres, type); - } } /* Save the new resource by chaining it into the head list for * the resource. If this is pass 2, we update any resource * pointers (currently only in the Job resource). */ -void save_resource(int type, RES_ITEM *items, int pass) +bool save_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { URES *res; + CONSRES *cons; int rindex = type - r_first; int i, size; int error = 0; @@ -445,10 +496,11 @@ void save_resource(int type, RES_ITEM *items, int pass) */ for (i=0; items[i].name; i++) { if (items[i].flags & ITEM_REQUIRED) { - if (!bit_is_set(i, res_all.res_dir.hdr.item_present)) { - Emsg2(M_ERROR_TERM, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + if (!bit_is_set(i, res_all.res_dir.hdr.item_present)) { + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), items[i].name, resources[rindex].name); - } + return false; + } } } @@ -466,14 +518,27 @@ void save_resource(int type, RES_ITEM *items, int pass) /* Resources containing another resource */ case R_DIRECTOR: if ((res = (URES *)GetResWithName(R_DIRECTOR, res_all.res_dir.hdr.name)) == NULL) { - Emsg1(M_ABORT, 0, _("Cannot find Director resource %s\n"), res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Director resource %s\n"), res_all.res_dir.hdr.name); + return false; } res->res_dir.tls_allowed_cns = res_all.res_dir.tls_allowed_cns; res->res_dir.disable_cmds = res_all.res_dir.disable_cmds; + res->res_dir.console = res_all.res_dir.console; + if (res_all.res_dir.remote && !res_all.res_dir.console) { + if ((cons = (CONSRES *)GetNextRes(R_CONSOLE, NULL)) == NULL) { + Mmsg(config->m_errmsg, _("Cannot find any Console resource for remote access\n")); + return false; + } + res->res_dir.console = cons; + } + break; + /* Resources containing another resource */ + case R_CONSOLE: break; case R_CLIENT: if ((res = (URES *)GetResWithName(R_CLIENT, res_all.res_dir.hdr.name)) == NULL) { - Emsg1(M_ABORT, 0, _("Cannot find Client resource %s\n"), res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Client resource %s\n"), res_all.res_dir.hdr.name); + return false; } res->res_client.pki_signing_key_files = res_all.res_client.pki_signing_key_files; res->res_client.pki_master_key_files = res_all.res_client.pki_master_key_files; @@ -500,7 +565,7 @@ void save_resource(int type, RES_ITEM *items, int pass) free(res_all.res_dir.hdr.desc); res_all.res_dir.hdr.desc = NULL; } - return; + return true; } /* The following code is only executed on pass 1 */ @@ -508,6 +573,9 @@ void save_resource(int type, RES_ITEM *items, int pass) case R_DIRECTOR: size = sizeof(DIRRES); break; + case R_CONSOLE: + size = sizeof(CONSRES); + break; case R_CLIENT: size = sizeof(CLIENT); break; @@ -522,31 +590,16 @@ void save_resource(int type, RES_ITEM *items, int pass) } /* Common */ if (!error) { - res = (URES *)malloc(size); - memcpy(res, &res_all, size); - if (!res_head[rindex]) { - res_head[rindex] = (RES *)res; /* store first entry */ - } else { - RES *next, *last; - /* Add new res to end of chain */ - for (last=next=res_head[rindex]; next; next=next->next) { - last = next; - if (strcmp(next->name, res->res_dir.hdr.name) == 0) { - Emsg2(M_ERROR_TERM, 0, - _("Attempt to define second \"%s\" resource named \"%s\" is not permitted.\n"), - resources[rindex].name, res->res_dir.hdr.name); - } - } - last->next = (RES *)res; - Dmsg2(90, "Inserting %s res: %s\n", res_to_str(type), - res->res_dir.hdr.name); + if (!config->insert_res(rindex, size)) { + return false; } } + return true; } bool parse_fd_config(CONFIG *config, const char *configfile, int exit_code) { config->init(configfile, NULL, exit_code, (void *)&res_all, res_all_size, - r_first, r_last, resources, res_head); + r_first, r_last, resources, &res_head); return config->parse_config(); } diff --git a/bacula/src/filed/filed_conf.h b/bacula/src/filed/filed_conf.h index ac04911e27..fea7efbd31 100644 --- a/bacula/src/filed/filed_conf.h +++ b/bacula/src/filed/filed_conf.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2016 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -30,8 +30,9 @@ #define R_DIRECTOR 1001 #define R_CLIENT 1002 #define R_MSGS 1003 +#define R_CONSOLE 1004 -#define R_LAST R_MSGS +#define R_LAST R_CONSOLE /* * Some resource attributes @@ -42,12 +43,34 @@ #define R_TYPE 1023 +/* Definition of the contents of each Resource */ +struct CONSRES { + RES hdr; + char *password; /* Director password */ + char *address; /* Director address or zero */ + int heartbeat_interval; + int comm_compression; + int32_t DIRport; + bool tls_authenticate; /* Authenticate with TSL */ + bool tls_enable; /* Enable TLS */ + bool tls_require; /* Require TLS */ + bool tls_verify_peer; /* TLS Verify Client Certificate */ + char *tls_ca_certfile; /* TLS CA Certificate File */ + char *tls_ca_certdir; /* TLS CA Certificate Directory */ + char *tls_certfile; /* TLS Server Certificate File */ + char *tls_keyfile; /* TLS Server Key File */ + char *tls_dhfile; /* TLS Diffie-Hellman Parameters */ + alist *tls_allowed_cns; /* TLS Allowed Clients */ + TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ +}; + /* Definition of the contents of each Resource */ struct DIRRES { RES hdr; char *password; /* Director password */ char *address; /* Director address or zero */ bool monitor; /* Have only access to status and .status functions */ + bool remote; /* Remote console, can run and control jobs */ bool tls_authenticate; /* Authenticate with TSL */ bool tls_enable; /* Enable TLS */ bool tls_require; /* Require TLS */ @@ -62,6 +85,7 @@ struct DIRRES { TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ alist *disable_cmds; /* Commands to disable */ bool *disabled_cmds_array; /* Disabled commands array */ + CONSRES *console; }; struct CLIENT { @@ -79,6 +103,7 @@ struct CLIENT { utime_t SDConnectTimeout; /* timeout in seconds */ utime_t heartbeat_interval; /* Interval to send heartbeats */ uint32_t max_network_buffer_size; /* max network buf size */ + bool comm_compression; /* Enable comm line compression */ bool pki_sign; /* Enable Data Integrity Verification via Digital Signatures */ bool pki_encrypt; /* Enable Data Encryption */ char *pki_keypair_file; /* PKI Key Pair File */ @@ -113,5 +138,6 @@ union URES { DIRRES res_dir; CLIENT res_client; MSGS res_msgs; + CONSRES res_cons; RES hdr; }; diff --git a/bacula/src/filed/heartbeat.c b/bacula/src/filed/heartbeat.c index c93e407677..2a8eeafd8b 100644 --- a/bacula/src/filed/heartbeat.c +++ b/bacula/src/filed/heartbeat.c @@ -93,12 +93,8 @@ extern "C" void *sd_heartbeat_thread(void *arg) } Dmsg2(200, "wait_intr=%d stop=%d\n", n, sd->is_stop()); } - /* - * Note, since sd and dir are local dupped sockets, this - * is one place where we can call destroy(). - */ - sd->destroy(); - dir->destroy(); + sd->close(); + dir->close(); jcr->hb_bsock = NULL; jcr->hb_started = false; jcr->hb_dir_bsock = NULL; @@ -192,7 +188,7 @@ extern "C" void *dir_heartbeat_thread(void *arg) } bmicrosleep(next, 0); } - dir->destroy(); + dir->close(); jcr->hb_bsock = NULL; jcr->hb_started = false; return NULL; diff --git a/bacula/src/filed/hello.c b/bacula/src/filed/hello.c index 87a31ba5e4..ce379ff816 100644 --- a/bacula/src/filed/hello.c +++ b/bacula/src/filed/hello.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Authenticate Director who is attempting to connect. * * Kern Sibbald, October 2000 - * */ #include "bacula.h" @@ -34,20 +33,21 @@ const int dbglvl = 50; * None prior to 10Mar08 * 1 10Mar08 * 2 13Mar09 - added the ability to restore from multiple storages - * 3 03Sep10 - added the restore object command + * 3 03Sep10 - added the restore object command for vss plugin 4.0 * 4 25Nov10 - added bandwidth command 5.1 - * 5 24Nov11 - added new restore object command format (pluginname) + * 5 24Nov11 - added new restore object command format (pluginname) 6.0 * 6 15Feb12 - added Component selection information list * 7 19Feb12 - added Expected files to restore * 8 22Mar13 - added restore options + version for SD * 9 06Aug13 - skipped - * 10 01Jan14 - added SD Calls Client + * 10 01Jan14 - added SD Calls Client and api version to status command * 11 O4May14 - skipped * 12 22Jun14 - skipped * 213 04Feb15 - added snapshot protocol with the DIR + * 214 20Mar17 - added comm line compression */ -#define FD_VERSION 213 /* FD version */ +#define FD_VERSION 214 /* FD version */ static char hello_sd[] = "Hello Bacula SD: Start Job %s %d\n"; static char hello_dir[] = "2000 OK Hello %d\n"; @@ -81,7 +81,8 @@ bool validate_dir_hello(JCR *jcr) dirname = check_pool_memory_size(dirname, dir->msglen); if (sscanf(dir->msg, "Hello Director %127s calling %d", dirname, &dir_version) != 2 && - sscanf(dir->msg, "Hello Director %127s calling", dirname) != 1) { + sscanf(dir->msg, "Hello Director %127s calling", dirname) != 1 && + sscanf(dir->msg, "Hello %127s calling %d", dirname, &dir_version) != 2 ) { char addr[64]; char *who = dir->get_peer(addr, sizeof(addr)) ? dir->who() : addr; dir->msg[100] = 0; @@ -91,6 +92,12 @@ bool validate_dir_hello(JCR *jcr) who, dir->msg); goto auth_fatal; } + if (dir_version >= 1 && me->comm_compression) { + dir->set_compress(); + } else { + dir->clear_compress(); + Dmsg0(050, "*** No FD compression to DIR\n"); + } unbash_spaces(dirname); foreach_res(director, R_DIRECTOR) { if (strcmp(director->hdr.name, dirname) == 0) @@ -168,6 +175,15 @@ void *handle_storage_connection(BSOCK *sd) return NULL; } + /* Turn on compression for newer FDs */ + if (sd_version >= 1 && me->comm_compression) { + sd->set_compress(); /* set compression allowed */ + } else { + sd->clear_compress(); + Dmsg2(050, "******** No FD compression to SD. sd_ver=%d compres=%d\n", + sd_version, me->comm_compression); + } + if (!jcr->max_bandwidth) { if (jcr->director->max_bandwidth_per_job) { jcr->max_bandwidth = jcr->director->max_bandwidth_per_job; @@ -178,6 +194,8 @@ void *handle_storage_connection(BSOCK *sd) } sd->set_bwlimit(jcr->max_bandwidth); + Dmsg1(200, "sd_version=%ld\n", sd_version); + pthread_cond_signal(&jcr->job_start_wait); /* wake waiting job */ free_jcr(jcr); return NULL; @@ -211,3 +229,141 @@ bool send_hello_sd(JCR *jcr, char *Job) Dmsg1(100, "Send to SD: %s\n", sd->msg); return rtn; } + +/* ======================== */ + +bool send_fdcaps(JCR *jcr, BSOCK *sd) { return false; } +bool recv_sdcaps(JCR *jcr) { return false; } + +/* Commands sent to Director */ +static char hello[] = "Hello %s calling %d\n"; + +/* Response from Director */ +static char DirOKhello[] = "1000 OK: %d"; +#define UA_VERSION 1 + +BSOCK *connect_director(JCR *jcr, CONSRES *dir) +{ + int tls_local_need = BNET_TLS_NONE; + int tls_remote_need = BNET_TLS_NONE; + bool tls_authenticate; + int compatible = true; + int dir_version = 0; + char bashed_name[MAX_NAME_LENGTH]; + char *password; + TLS_CONTEXT *tls_ctx = NULL; + BSOCK *UA_sock = NULL; + int heart_beat; + + if (!dir) { + return 0; + } + + Dmsg2(0, "Connecting to Director %s:%d\n", dir->address, dir->DIRport); + + if (dir) { + heart_beat = dir->heartbeat_interval; + } else { + heart_beat = 0; + } + UA_sock = new_bsock(); + if (!UA_sock->connect(NULL, 5, 15, heart_beat, "Director daemon", dir->address, + NULL, dir->DIRport, 0)) { + free_bsock(UA_sock); + return NULL; + } + + /* + * Send my name to the Director then do authentication + */ + bstrncpy(bashed_name, dir->hdr.name, sizeof(bashed_name)); + bash_spaces(bashed_name); + password = dir->password; + /* TLS Requirement */ + if (dir->tls_enable) { + if (dir->tls_require) { + tls_local_need = BNET_TLS_REQUIRED; + } else { + tls_local_need = BNET_TLS_OK; + } + } + if (dir->tls_authenticate) { + tls_local_need = BNET_TLS_REQUIRED; + } + tls_authenticate = dir->tls_authenticate; + tls_ctx = dir->tls_ctx; + + /* Timeout Hello after 15 secs */ + btimer_t *tid = start_bsock_timer(UA_sock, 15); + UA_sock->fsend(hello, bashed_name, UA_VERSION); + + if (!cram_md5_respond(UA_sock, password, &tls_remote_need, &compatible) || + !cram_md5_challenge(UA_sock, password, tls_local_need, compatible)) { + goto bail_out; + } + + /* Verify that the remote host is willing to meet our TLS requirements */ + if (tls_remote_need < tls_local_need && tls_local_need != BNET_TLS_OK && tls_remote_need != BNET_TLS_OK) { + Mmsg(jcr->errmsg, _("Authorization problem:" + " Remote server did not advertise required TLS support.\n")); + goto bail_out; + } + + /* Verify that we are willing to meet the remote host's requirements */ + if (tls_remote_need > tls_local_need && tls_local_need != BNET_TLS_OK && tls_remote_need != BNET_TLS_OK) { + Mmsg(jcr->errmsg, _("Authorization problem:" + " Remote server requires TLS.\n")); + goto bail_out; + } + + /* Is TLS Enabled? */ + if (tls_local_need >= BNET_TLS_OK && tls_remote_need >= BNET_TLS_OK) { + /* Engage TLS! Full Speed Ahead! */ + if (!bnet_tls_client(tls_ctx, UA_sock, NULL)) { + Mmsg(jcr->errmsg, _("TLS negotiation failed\n")); + goto bail_out; + } + if (tls_authenticate) { /* Authenticate only? */ + UA_sock->free_tls(); /* yes, shutdown tls */ + } + } + + /* + * It's possible that the TLS connection will + * be dropped here if an invalid client certificate was presented + */ + Dmsg1(6, ">dird: %s", UA_sock->msg); + if (UA_sock->recv() <= 0) { + Mmsg(jcr->errmsg, _("Bad response to Hello command: ERR=%s\n"), + UA_sock->bstrerror()); + goto bail_out; + } + + Dmsg1(10, "msg); + if (strncmp(UA_sock->msg, DirOKhello, sizeof(DirOKhello)-3) == 0) { + sscanf(UA_sock->msg, DirOKhello, &dir_version); + Dmsg1(0, "%s\n", UA_sock->msg); + + } else { + Mmsg(jcr->errmsg, _("Director rejected Hello command\n")); + goto bail_out; + } + /* Turn on compression for newer Directors */ + if (dir_version >= 1 && (!dir || dir->comm_compression)) { + UA_sock->set_compress(); + } else { + UA_sock->clear_compress(); + } + stop_bsock_timer(tid); + return UA_sock; + +bail_out: + free_bsock(UA_sock); + stop_bsock_timer(tid); + Mmsg(jcr->errmsg, + ( _("Director authorization problem.\n" + "Most likely the passwords do not agree.\n" + "If you are using TLS, there may have been a certificate validation error during the TLS handshake.\n" + "For help, please see " MANUAL_AUTH_URL "\n"))); + return NULL; +} diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index a5974e24d2..81cef24486 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Bacula File Daemon Job processing * * Written by Kern Sibbald, October MM - * */ #include "bacula.h" @@ -33,15 +32,13 @@ bool no_win32_write_errors = false; /* Static variables */ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; - -const bool have_win32 = false; #ifdef HAVE_ACL const bool have_acl = true; #else const bool have_acl = false; #endif - + #if HAVE_XATTR const bool have_xattr = true; #else @@ -82,21 +79,25 @@ static int restore_object_cmd(JCR *jcr); static int set_options(findFOPTS *fo, const char *opts); static void set_storage_auth_key(JCR *jcr, char *key); static int sm_dump_cmd(JCR *jcr); +static int proxy_cmd(JCR *jcr); +static int fd_testnetwork_cmd(JCR *jcr); #ifdef DEVELOPER static int exit_cmd(JCR *jcr); #endif /* Exported functions */ +#define ACCESS_MONITOR 1 +#define ACCESS_REMOTE 2 /* * The following are the recognized commands from the Director. */ struct s_cmds cmds[] = { {"backup", backup_cmd, 0}, - {"cancel", cancel_cmd, 0}, + {"cancel", cancel_cmd, ACCESS_REMOTE}, {"setdebug=", setdebug_cmd, 0}, - {"setbandwidth=",setbandwidth_cmd, 0}, + {"setbandwidth=",setbandwidth_cmd, ACCESS_REMOTE}, {"snapshot", snapshot_cmd, 0}, {"estimate", estimate_cmd, 0}, {"Hello", hello_cmd, 1}, @@ -106,8 +107,8 @@ struct s_cmds cmds[] = { {"restore ", restore_cmd, 0}, {"endrestore", end_restore_cmd, 0}, {"session", session_cmd, 0}, - {"status", status_cmd, 1}, - {".status", qstatus_cmd, 1}, + {"status", status_cmd, ACCESS_MONITOR|ACCESS_REMOTE}, + {".status", qstatus_cmd, ACCESS_MONITOR|ACCESS_REMOTE}, {"storage ", storage_cmd, 0}, {"verify", verify_cmd, 0}, {"component", component_cmd, 0}, @@ -118,7 +119,9 @@ struct s_cmds cmds[] = { {"accurate", accurate_cmd, 0}, {"restoreobject", restore_object_cmd, 0}, {"sm_dump", sm_dump_cmd, 0}, - {"stop", cancel_cmd, 0}, + {"stop", cancel_cmd, ACCESS_REMOTE}, + {"proxy", proxy_cmd, ACCESS_REMOTE}, + {"testnetwork", fd_testnetwork_cmd, 0}, #ifdef DEVELOPER {"exit", exit_cmd, 0}, #endif @@ -175,7 +178,7 @@ static char OKsetdebug[] = "2000 OK setdebug=%ld trace=%ld hangup=%ld" static char BADjob[] = "2901 Bad Job\n"; static char EndJob[] = "2800 End Job TermCode=%d JobFiles=%d ReadBytes=%lld" " JobBytes=%lld Errors=%d VSS=%d Encrypt=%d" - " CommBytes=0 CompressCommBytes=0\n"; + " CommBytes=%lld CompressCommBytes=%lld\n"; static char OKRunBefore[] = "2000 OK RunBefore\n"; static char OKRunBeforeNow[] = "2000 OK RunBeforeNow\n"; static char OKRunAfter[] = "2000 OK RunAfter\n"; @@ -203,6 +206,21 @@ static char read_data[] = "read data %d\n"; static char read_close[] = "read close session %d\n"; static char read_ctrl[] = "read control %d\n"; +/* Should tell us if a command is authorized or not */ +static bool access_ok(struct s_cmds *cmd, DIRRES* dir) +{ + if ((cmd->access & ACCESS_MONITOR) && dir->monitor) { + return true; + } + if ((cmd->access & ACCESS_REMOTE) && dir->remote) { + return true; + } + if (!dir->remote && !dir->monitor) { + return true; + } + return false; +} + /* * Accept requests from a Director * @@ -295,7 +313,7 @@ static void *handle_director_request(BSOCK *dir) dir->signal(BNET_EOD); break; } - if ((jcr->authenticated) && (!cmds[i].monitoraccess) && (jcr->director->monitor)) { + if (jcr->authenticated && !access_ok(&cmds[i], jcr->director)) { Dmsg1(100, "Command \"%s\" is invalid.\n", cmds[i].cmd); dir->fsend(invalid_cmd); dir->signal(BNET_EOD); @@ -331,21 +349,33 @@ static void *handle_director_request(BSOCK *dir) /* Run the after job */ run_scripts(jcr, jcr->RunScripts, "ClientAfterJob"); + /* send any queued messages before reporting the jobstatus to the director */ + dequeue_messages(jcr); + if (jcr->JobId) { /* send EndJob if running a job */ + uint64_t CommBytes, CommCompressedBytes; uint32_t vss, encrypt; /* Send termination status back to Dir */ + if (jcr->store_bsock) { + CommBytes = jcr->store_bsock->CommBytes(); + CommCompressedBytes = jcr->store_bsock->CommCompressedBytes(); + } else { + CommBytes = CommCompressedBytes = 0; + } encrypt = jcr->crypto.pki_encrypt; vss = jcr->Snapshot; dir->fsend(EndJob, jcr->JobStatus, jcr->JobFiles, jcr->ReadBytes, jcr->JobBytes, jcr->JobErrors, vss, - encrypt, 0, 0); + encrypt, CommBytes, CommCompressedBytes); + //Dmsg0(0, dir->msg); } generate_daemon_event(jcr, "JobEnd"); generate_plugin_event(jcr, bEventJobEnd); bail_out: - dequeue_messages(jcr); /* send any queued messages */ + dequeue_messages(jcr); /* send any queued messages, will no longer impact + * the job status... */ /* Inform Director that we are done */ dir->signal(BNET_TERMINATE); @@ -452,13 +482,12 @@ void *handle_connection_request(void *caller) if (bs->msglen < 25 || bs->msglen > 500) { goto bail_out; } - Dmsg1(100, "Got: %s", bs->msg); - if (strncmp(bs->msg, "Hello Director", 14) == 0) { - return handle_director_request(bs); - } if (strncmp(bs->msg, "Hello FD: Bacula Storage", 20) ==0) { return handle_storage_connection(bs); } + if (strncmp(bs->msg, "Hello ", 5) == 0) { + return handle_director_request(bs); + } } bail_out: Dmsg2(100, "Bad command from %s. Len=%d.\n", bs->who(), bs->msglen); @@ -469,6 +498,183 @@ bail_out: return NULL; } + +/* + * Test the Network between FD/SD + */ +static int fd_testnetwork_cmd(JCR *jcr) +{ + bool can_compress, ok=true; + BSOCK *sd = jcr->store_bsock; + int64_t nb=0, nb2=0; + char ed1[50]; + btime_t start, end; + + if (!sd || !jcr->dir_bsock) { + return 1; + } + if (sscanf(jcr->dir_bsock->msg, "testnetwork bytes=%lld", &nb) != 1 || nb <= 0) { + sd->fsend("2999 testnetwork command error\n"); + return 1; + } + + /* We disable the comline compression, else all numbers will be wrong */ + can_compress = sd->can_compress(); + + sd->fsend("testnetwork bytes=%lld\n", nb); + sd->clear_compress(); + + /* In the first step, we send X bytes to the SD */ + memset(sd->msg, 0xAA, sizeof_pool_memory(sd->msg)); + sd->msglen = sizeof_pool_memory(sd->msg); + + start = get_current_btime(); + for (nb2 = nb ; nb2 > 0 && ok ; nb2 -= sd->msglen) { + if (nb2 < sd->msglen) { + sd->msglen = nb2; + } + ok = sd->send(); + } + sd->signal(BNET_EOD); + end = get_current_btime() + 1; + + if (!ok) { + goto bail_out; + } + + jcr->dir_bsock->fsend("2000 OK bytes=%lld duration=%lldms write_speed=%sB/s\n", + nb, end/1000 - start/1000, + edit_uint64_with_suffix(nb * 1000000 / (end - start), ed1)); + + /* Now we receive X bytes from the SD */ + start = get_current_btime(); + for (nb2 = 0; sd->recv() > 0; nb2 += sd->msglen) { } + end = get_current_btime() + 1; + + jcr->dir_bsock->fsend("2000 OK bytes=%lld duration=%lldms read_speed=%sB/s\n", + nb2, end/1000 - start/1000, + edit_uint64_with_suffix(nb2 * 1000000 / (end - start), ed1)); + + jcr->dir_bsock->signal(BNET_CMD_OK); + +bail_out: + if (can_compress) { + sd->set_compress(); + } + if (!ok) { + jcr->dir_bsock->fsend("2999 network test failed ERR=%s\n", sd->errmsg); + jcr->dir_bsock->signal(BNET_CMD_FAILED); + } + + return 1; +} + +static int proxy_cmd(JCR *jcr) +{ + bool OK=true, fdcalled = false; + BSOCK *cons_bsock; + CONSRES *cons = jcr->director->console; + int v, maxfd; + fd_set fdset; + struct timeval tv; + + if (!cons) { + cons = (CONSRES *)GetNextRes(R_CONSOLE, NULL); + } + /* Here, dir_bsock is not really the director, this is a console */ + cons_bsock = connect_director(jcr, cons); + if (!cons_bsock) { + jcr->dir_bsock->signal(BNET_ERROR_MSG); + jcr->dir_bsock->fsend("2999 proxy error. ERR=%s\n", jcr->errmsg); + jcr->dir_bsock->signal(BNET_MAIN_PROMPT); + /* Error during the connect */ + return 1; + } + + /* Inform the console that the command is OK */ + jcr->dir_bsock->fsend("2000 proxy OK.\n"); + jcr->dir_bsock->signal(BNET_MAIN_PROMPT); + + maxfd = MAX(cons_bsock->m_fd, jcr->dir_bsock->m_fd) + 1; + + /* Start to forward events from one to the other + * It can be done with 2 threads, or with a select + */ + do { + FD_ZERO(&fdset); + FD_SET((unsigned)cons_bsock->m_fd, &fdset); + FD_SET((unsigned)jcr->dir_bsock->m_fd, &fdset); + + tv.tv_sec = 5; + tv.tv_usec = 0; + switch ((v = select(maxfd, &fdset, NULL, NULL, &tv))) { + case 0: /* timeout */ + OK = !jcr->is_canceled(); + break; + case -1: + Dmsg1(0, "Bad call to select ERR=%d\n", errno); + OK = false; + default: +#ifdef HAVE_TLS + if (cons_bsock->tls && !tls_bsock_probe(cons_bsock)) { + /* maybe a session key negotiation waked up the socket */ + FD_CLR(cons_bsock->m_fd, &fdset); + } + if (jcr->dir_bsock->tls && !tls_bsock_probe(jcr->dir_bsock)) { + /* maybe a session key negotiation waked up the socket */ + FD_CLR(jcr->dir_bsock->m_fd, &fdset); + } +#endif + break; + } + Dmsg1(DT_NETWORK, "select = %d\n", v); + if (OK) { + if (FD_ISSET(cons_bsock->m_fd, &fdset)) { + v = cons_bsock->recv(); + if (v == BNET_SIGNAL) { + if (cons_bsock->msglen == BNET_FDCALLED) { + OK = false; + fdcalled = true; + } else { + jcr->dir_bsock->signal(cons_bsock->msglen); + } + + } else if (v >= 0) { + jcr->dir_bsock->fsend("%s", cons_bsock->msg); + + } else { + /* We should not have such kind of message */ + OK = false; + } + } + if (FD_ISSET(jcr->dir_bsock->m_fd, &fdset)) { + v = jcr->dir_bsock->recv(); + if (v == BNET_SIGNAL) { + cons_bsock->signal(jcr->dir_bsock->msglen); + } else if (v >= 0) { + cons_bsock->fsend("%s", jcr->dir_bsock->msg); + } else { + /* We should not have such kind of message */ + OK = false; + } + } + } + if (cons_bsock->is_error() || jcr->dir_bsock->is_error()) { + OK = false; + } + } while (OK && !jcr->is_canceled()); + + /* Close the socket, nothing more will come */ + jcr->dir_bsock->signal(BNET_TERMINATE); + jcr->dir_bsock->close(); + if (fdcalled) { + handle_connection_request(cons_bsock); /* will release the socket */ + } else { + free_bsock(cons_bsock); + } + return 1; +} + static int sm_dump_cmd(JCR *jcr) { close_memory_pool(); @@ -486,8 +692,7 @@ static int exit_cmd(JCR *jcr) } #endif - -/** +/* * Hello from Director he must identify himself and provide his * password. */ @@ -503,10 +708,11 @@ static int hello_cmd(JCR *jcr) Dmsg0(120, "OK Authenticate\n"); jcr->authenticated = true; + dequeue_messages(jcr); /* dequeue any daemon messages */ return 1; } -/** +/* * Cancel a Job */ static int cancel_cmd(JCR *jcr) @@ -533,8 +739,7 @@ static int cancel_cmd(JCR *jcr) generate_plugin_event(cjcr, bEventCancelCommand, NULL); cjcr->setJobStatus(status); if (cjcr->store_bsock) { - cjcr->store_bsock->set_timed_out(); - cjcr->store_bsock->set_terminated(); + cjcr->store_bsock->cancel(); } cjcr->my_thread_send_signal(TIMEOUT_SIGNAL); free_jcr(cjcr); @@ -636,6 +841,20 @@ static int setdebug_cmd(JCR *jcr) } debug_level_tags = level_tags; + /* Parse specific FD options */ + for (char *p = options; *p ; p++) { + switch(*p) { + case 'i': + /* Turn on/off ignore bwrite() errors on restore */ + no_win32_write_errors = true; + break; + case 'd': + /* Turn on/off decomp of BackupRead() streams */ + win32decomp = true; + break; + } + } + /* handle other options */ set_debug_flags(options); @@ -690,14 +909,16 @@ static int job_cmd(JCR *jcr) return dir->fsend(OKjob, VERSION, LSMDATE, HOST_OS, DISTNAME, DISTVER); } -extern "C" char *job_code_callback_filed(JCR *jcr, const char* param) +extern "C" char *job_code_callback_filed(JCR *jcr, const char* param, char *buf, int buflen) { switch (param[0]) { - case 'D': - if (jcr->director) { - return jcr->director->hdr.name; - } - break; + case 'D': + if (jcr->director) { + return jcr->director->hdr.name; + } + break; + case 'S': + return jcr->PrevJob; } return NULL; @@ -948,17 +1169,14 @@ static bool init_fileset(JCR *jcr) return true; } - static void append_file(JCR *jcr, findINCEXE *incexe, const char *buf, bool is_file) { if (is_file) { incexe->name_list.append(new_dlistString(buf)); - } else if (me->plugin_directory) { generate_plugin_event(jcr, bEventPluginCommand, (void *)buf); incexe->plugin_list.append(new_dlistString(buf)); - } else { Jmsg(jcr, M_FATAL, 0, _("Plugin Directory not defined. Cannot use plugin: \"%s\"\n"), @@ -1830,7 +2048,7 @@ static void set_storage_auth_key(JCR *jcr, char *key) } jcr->sd_auth_key = bstrdup(key); - Dmsg1(5, "set sd auth key %s\n", jcr->sd_auth_key); + Dmsg1(200, "set sd auth key %s\n", jcr->sd_auth_key); } /** @@ -1956,7 +2174,7 @@ bail_out: } -/** +/* * Do a backup. */ static int backup_cmd(JCR *jcr) @@ -1976,7 +2194,7 @@ static int backup_cmd(JCR *jcr) * If explicitly requesting FO_ACL or FO_XATTR, fail job if it * is not available on Client machine */ - if (jcr->ff->flags & FO_ACL && !(have_acl||have_win32)) { + if (jcr->ff->flags & FO_ACL && !(have_acl)) { Jmsg(jcr, M_FATAL, 0, _("ACL support not configured for Client.\n")); goto cleanup; } @@ -1996,7 +2214,7 @@ static int backup_cmd(JCR *jcr) dir->fsend(OKbackup); Dmsg1(110, "filed>dird: %s", dir->msg); - /** + /* * Send Append Open Session to Storage daemon */ sd->fsend(append_open); @@ -2041,8 +2259,7 @@ static int backup_cmd(JCR *jcr) /* Call RunScript just after the Snapshot creation, usually, we restart services */ run_scripts(jcr, jcr->RunScripts, "ClientAfterVSS"); - - /** + /* * Send Files to Storage daemon */ Dmsg1(110, "begin blast ff=%p\n", (FF_PKT *)jcr->ff); @@ -2079,18 +2296,14 @@ static int backup_cmd(JCR *jcr) * Send Append Close to Storage daemon */ sd->fsend(append_close, jcr->Ticket); - sd->msg[0] = 0; while (bget_msg(sd) >= 0) { /* stop on signal or error */ if (sscanf(sd->msg, OK_close, &SDJobStatus) == 1) { ok = 1; Dmsg2(200, "SDJobStatus = %d %c\n", SDJobStatus, (char)SDJobStatus); - } else { - Dmsg1(100, "append_close: scan fail from %s\n", sd->msg); } } if (!ok) { Jmsg(jcr, M_FATAL, 0, _("Append Close with SD failed.\n")); - Dmsg1(100, "append_close: scan fail from %s\n", sd->msg); goto cleanup; } if (!(SDJobStatus == JS_Terminated || SDJobStatus == JS_Warnings || @@ -2101,7 +2314,6 @@ static int backup_cmd(JCR *jcr) } cleanup: - generate_plugin_event(jcr, bEventEndBackupJob); return 0; /* return and stop command loop */ } @@ -2303,7 +2515,10 @@ static int restore_cmd(JCR *jcr) generate_daemon_event(jcr, "JobStart"); generate_plugin_event(jcr, bEventStartRestoreJob); - do_restore(jcr); + if (!jcr->is_canceled()) { + do_restore(jcr); + } + stop_dir_heartbeat(jcr); jcr->setJobStatus(JS_Terminated); @@ -2445,9 +2660,9 @@ static void filed_free_jcr(JCR *jcr) delete jcr->RunScripts; free_path_list(jcr); - if (jcr->JobId != 0) + if (jcr->JobId != 0) { write_state_file(me->working_directory, "bacula-fd", get_first_port_host_order(me->FDaddrs)); - + } return; } diff --git a/bacula/src/filed/protos.h b/bacula/src/filed/protos.h index 155432bde3..ed2e5fe431 100644 --- a/bacula/src/filed/protos.h +++ b/bacula/src/filed/protos.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -18,7 +18,6 @@ */ /* * Written by Kern Sibbald, MM - * */ extern bool blast_data_to_storage_daemon(JCR *jcr, char *addr); @@ -39,8 +38,7 @@ bool send_hello_ok(BSOCK *bs); bool send_sorry(BSOCK *bs); bool send_hello_sd(JCR *jcr, char *Job); void *handle_storage_connection(BSOCK *sd); -bool send_fdcaps(JCR *jcr); -bool recv_sdcaps(JCR *jcr); +BSOCK *connect_director(JCR *jcr, CONSRES *dir); /* From verify.c */ int digest_file(JCR *jcr, FF_PKT *ff_pkt, DIGEST *digest); @@ -57,6 +55,7 @@ bool accurate_finish(JCR *jcr); bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt); bool accurate_mark_file_as_seen(JCR *jcr, char *fname); void accurate_free(JCR *jcr); +bool accurate_check_file(JCR *jcr, ATTR *attr, char *digest); /* from backup.c */ void strip_path(FF_PKT *ff_pkt); diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c index f7ee5a6d00..7e5a3c8e25 100644 --- a/bacula/src/filed/restore.c +++ b/bacula/src/filed/restore.c @@ -2,7 +2,6 @@ Bacula(R) - The Network Backup Solution Copyright (C) 2000-2017 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -21,7 +20,6 @@ * Bacula File Daemon restore.c Restorefiles. * * Kern Sibbald, November MM - * */ #include "bacula.h" @@ -355,6 +353,7 @@ void do_restore(JCR *jcr) int64_t rsrc_len = 0; /* Original length of resource fork */ r_ctx rctx; ATTR *attr; + int bget_ret = 0; /* ***FIXME*** make configurable */ crypto_digest_t signing_algorithm = have_sha2 ? CRYPTO_DIGEST_SHA256 : CRYPTO_DIGEST_SHA1; @@ -395,7 +394,6 @@ void do_restore(JCR *jcr) jcr->compress_buf_size = compress_buf_size; } - GetMsg *fdmsg; fdmsg = New(GetMsg(jcr, sd, rec_header, GETMSG_MAX_MSG_SIZE)); @@ -451,7 +449,17 @@ void do_restore(JCR *jcr) jcr->xacl = (XACL*)new_xacl(); Dsm_check(200); - while (fdmsg->bget_msg(&bmsg) >= 0 && !job_canceled(jcr)) { + while ((bget_ret = fdmsg->bget_msg(&bmsg)) >= 0 && !job_canceled(jcr)) { + time_t now = time(NULL); + if (jcr->last_stat_time == 0) { + jcr->last_stat_time = now; + jcr->stat_interval = 30; /* Default 30 seconds */ + } else if (now >= jcr->last_stat_time + jcr->stat_interval) { + jcr->dir_bsock->fsend("Progress JobId=x files=%ld bytes=%lld bps=%ld\n", + jcr->JobFiles, jcr->JobBytes, jcr->LastRate); + jcr->last_stat_time = now; + } + /* Remember previous stream type */ rctx.prev_stream = rctx.stream; @@ -466,8 +474,12 @@ void do_restore(JCR *jcr) rctx.stream = rctx.full_stream & STREAMMASK_TYPE; /* Now we expect the Stream Data */ - if (fdmsg->bget_msg(&bmsg) < 0) { - Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror()); + if ((bget_ret = fdmsg->bget_msg(&bmsg)) < 0) { + if (bget_ret != BNET_EXT_TERMINATE) { + Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror()); + } else { + /* The error has been handled somewhere else, just quit */ + } goto get_out; } if (rctx.size != (uint32_t)bmsg->origlen) { @@ -477,8 +489,6 @@ void do_restore(JCR *jcr) bmsg->origlen, rctx.size); goto get_out; } - Dmsg3(620, "Got stream: %s len=%d extract=%d\n", stream_to_ascii(rctx.stream), - bmsg->msglen, rctx.extract); /* If we change streams, close and reset alternate data streams */ if (rctx.prev_stream != rctx.stream) { @@ -601,6 +611,7 @@ void do_restore(JCR *jcr) } break; } + break; /* Data stream */ @@ -700,7 +711,8 @@ void do_restore(JCR *jcr) if (rctx.stream == STREAM_SPARSE_DATA || rctx.stream == STREAM_SPARSE_COMPRESSED_DATA - || rctx.stream == STREAM_SPARSE_GZIP_DATA) { + || rctx.stream == STREAM_SPARSE_GZIP_DATA) + { rctx.flags |= FO_SPARSE; } @@ -980,9 +992,18 @@ void do_restore(JCR *jcr) Dmsg2(0, "Unknown stream=%d data=%s\n", rctx.stream, bmsg->rbuf); break; } /* end switch(stream) */ + + /* Debug code: check if we must hangup or blowup */ + if (handle_hangup_blowup(jcr, jcr->JobFiles, jcr->JobBytes)) { + goto get_out; + } + Dsm_check(200); - } /* end while get_msg() */ + } /* end while bufmsg->bget_msg(&bmsg)) */ + if (bget_ret == BNET_EXT_TERMINATE) { + goto get_out; + } /* * If output file is still open, it was the last one in the * archive since we just hit an end of file, so close the file. @@ -1002,7 +1023,7 @@ get_out: ok_out: Dsm_check(200); - fdmsg->wait_read_sock(); + fdmsg->wait_read_sock(jcr->is_job_canceled()); delete bmsg; free_GetMsg(fdmsg); Dsm_check(200); @@ -1126,14 +1147,15 @@ static int do_file_digest(JCR *jcr, FF_PKT *ff_pkt, bool top_level) return (digest_file(jcr, ff_pkt, jcr->crypto.digest)); } -bool sparse_data(JCR *jcr, BFILE *bfd, uint64_t *addr, char **data, uint32_t *length) +bool sparse_data(JCR *jcr, BFILE *bfd, uint64_t *addr, char **data, uint32_t *length, int flags) { unser_declare; uint64_t faddr; char ec1[50]; unser_begin(*data, OFFSET_FADDR_SIZE); unser_uint64(faddr); - if (*addr != faddr) { + /* We seek only if we have a SPARSE stream, not for OFFSET */ + if ((flags & FO_SPARSE) && *addr != faddr) { *addr = faddr; if (blseek(bfd, (boffset_t)*addr, SEEK_SET) < 0) { berrno be; @@ -1399,7 +1421,7 @@ int32_t extract_data(r_ctx &rctx, POOLMEM *buf, int32_t buflen) } if ((flags & FO_SPARSE) || (flags & FO_OFFSETS)) { - if (!sparse_data(jcr, bfd, &rctx.fileAddr, &wbuf, &wsize)) { + if (!sparse_data(jcr, bfd, &rctx.fileAddr, &wbuf, &wsize, flags)) { goto get_out; } } @@ -1530,7 +1552,7 @@ again: Dmsg2(130, "Encryption writing full block, %u bytes, remaining %u bytes in buffer\n", wsize, cipher_ctx->buf_len); if ((flags & FO_SPARSE) || (flags & FO_OFFSETS)) { - if (!sparse_data(jcr, bfd, addr, &wbuf, &wsize)) { + if (!sparse_data(jcr, bfd, addr, &wbuf, &wsize, flags)) { return false; } } diff --git a/bacula/src/filed/restore.h b/bacula/src/filed/restore.h index e04ecd3632..2d0ec0ba9c 100644 --- a/bacula/src/filed/restore.h +++ b/bacula/src/filed/restore.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2009-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. diff --git a/bacula/src/filed/status.c b/bacula/src/filed/status.c index b753b58d6a..2a9deaa112 100644 --- a/bacula/src/filed/status.c +++ b/bacula/src/filed/status.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Bacula File Daemon Status routines * * Kern Sibbald, August MMI - * */ #include "bacula.h" @@ -43,8 +42,6 @@ static char qstatus2[] = ".status %127s api=%d api_opts=%127s"; static char OKqstatus[] = "2000 OK .status\n"; static char DotStatusJob[] = "JobId=%d JobStatus=%c JobErrors=%d\n"; -#define VSS "" - /* * General status generator */ @@ -62,15 +59,46 @@ static const bool have_lzo = false; #endif +static void api_list_status_header(STATUS_PKT *sp) +{ + char *p; + char buf[300]; + OutputWriter wt(sp->api_opts); + *buf = 0; + + wt.start_group("header"); + wt.get_output( + OT_STRING, "name", my_name, + OT_STRING, "version", VERSION " (" BDATE ")", + OT_STRING, "uname", HOST_OS " " DISTNAME " " DISTVER, + OT_UTIME, "started", daemon_start_time, + OT_INT, "jobs_run", num_jobs_run, + OT_INT, "jobs_running",job_count(), + OT_STRING, "winver", buf, + OT_INT64, "debug", debug_level, + OT_INT, "trace", get_trace(), + OT_INT64, "bwlimit", me->max_bandwidth_per_job, + OT_PLUGINS, "plugins", b_plugin_list, + OT_END); + p = wt.end_group(); + sendit(p, strlen(p), sp); +} + static void list_status_header(STATUS_PKT *sp) { POOL_MEM msg(PM_MESSAGE); char b1[32], b2[32], b3[32], b4[32], b5[35]; + int64_t memused = (char *)sbrk(0)-(char *)start_heap; int len; char dt[MAX_TIME_LENGTH]; - len = Mmsg(msg, _("%s %sVersion: %s (%s) %s %s %s %s\n"), - my_name, "", VERSION, BDATE, VSS, HOST_OS, + if (sp->api) { + api_list_status_header(sp); + return; + } + + len = Mmsg(msg, _("%s %sVersion: %s (%s) %s %s %s\n"), + my_name, "", VERSION, BDATE, HOST_OS, DISTNAME, DISTVER); sendit(msg.c_str(), len, sp); bstrftime_nc(dt, sizeof(dt), daemon_start_time); @@ -78,16 +106,16 @@ static void list_status_header(STATUS_PKT *sp) dt, num_jobs_run, job_count()); sendit(msg.c_str(), len, sp); len = Mmsg(msg, _(" Heap: heap=%s smbytes=%s max_bytes=%s bufs=%s max_bufs=%s\n"), - edit_uint64_with_commas((char *)sbrk(0)-(char *)start_heap, b1), + edit_uint64_with_commas(memused, b1), edit_uint64_with_commas(sm_bytes, b2), edit_uint64_with_commas(sm_max_bytes, b3), edit_uint64_with_commas(sm_buffers, b4), edit_uint64_with_commas(sm_max_buffers, b5)); sendit(msg.c_str(), len, sp); len = Mmsg(msg, _(" Sizes: boffset_t=%d size_t=%d debug=%s trace=%d " - "mode=%d bwlimit=%skB/s\n"), + "mode=%d,%d bwlimit=%skB/s\n"), sizeof(boffset_t), sizeof(size_t), - edit_uint64(debug_level, b2), get_trace(), (int)DEVELOPER_MODE, + edit_uint64(debug_level, b2), get_trace(), (int)DEVELOPER_MODE, 0, edit_uint64_with_commas(me->max_bandwidth_per_job/1024, b1)); sendit(msg.c_str(), len, sp); if (b_plugin_list && b_plugin_list->size() > 0) { @@ -132,7 +160,6 @@ static void list_running_jobs_plain(STATUS_PKT *sp) Dmsg0(1000, "Begin status jcr loop.\n"); len = Mmsg(msg, _("\nRunning Jobs:\n")); sendit(msg.c_str(), len, sp); - const char *vss = ""; foreach_jcr(njcr) { bstrftime_nc(dt, sizeof(dt), njcr->start_time); if (njcr->JobId == 0) { @@ -143,8 +170,8 @@ static void list_running_jobs_plain(STATUS_PKT *sp) len = Mmsg(msg, _("JobId %d Job %s is running.\n"), njcr->JobId, njcr->Job); sendit(msg.c_str(), len, sp); - len = Mmsg(msg, _(" %s%s %s Job started: %s\n"), - vss, job_level_to_str(njcr->getJobLevel()), + len = Mmsg(msg, _(" %s %s Job started: %s\n"), + job_level_to_str(njcr->getJobLevel()), job_type_to_str(njcr->getJobType()), dt); } sendit(msg.c_str(), len, sp); @@ -225,9 +252,93 @@ static void list_running_jobs_plain(STATUS_PKT *sp) sendit(_("====\n"), 5, sp); } +/* + * List running jobs for Bat or Bweb in a format + * simpler to parse. Be careful when changing this + * subroutine. + */ +static void list_running_jobs_api(STATUS_PKT *sp) +{ + OutputWriter ow(sp->api_opts); + int sec, bps; + char *p; + JCR *njcr; + + /* API v1, edit with comma, space before the name, sometime ' ' as separator */ + + foreach_jcr(njcr) { + p = ow.get_output(OT_CLEAR, OT_START_OBJ, OT_END); + + if (njcr->JobId == 0) { + ow.get_output(OT_UTIME, "DirectorConnected", njcr->start_time, + OT_INT, "DirTLS", (njcr->dir_bsock && njcr->dir_bsock->tls)?1:0, + OT_END); + } else { + ow.get_output(OT_INT32, "JobId", njcr->JobId, + OT_STRING, "Job", njcr->Job, + OT_JOBLEVEL,"Level", njcr->getJobLevel(), + OT_JOBTYPE, "Type", njcr->getJobType(), + OT_JOBSTATUS, "Status", njcr->getJobStatus(), + OT_UTIME, "StartTime", njcr->start_time, + OT_END); + + } + sendit(p, strlen(p), sp); + if (njcr->JobId == 0) { + continue; + } + sec = time(NULL) - njcr->start_time; + if (sec <= 0) { + sec = 1; + } + bps = (int)(njcr->JobBytes / sec); + ow.get_output(OT_CLEAR, + OT_INT32, "JobFiles", njcr->JobFiles, + OT_SIZE, "JobBytes", njcr->JobBytes, + OT_INT, "Bytes/sec", bps, + OT_INT, "Errors", njcr->JobErrors, + OT_INT64, "Bwlimit", njcr->max_bandwidth, + OT_SIZE, "ReadBytes", njcr->ReadBytes, + OT_END); + + ow.get_output(OT_INT32, "Files Examined", njcr->num_files_examined, OT_END); + + if (njcr->is_JobType(JT_RESTORE) && njcr->ExpectedFiles > 0) { + ow.get_output(OT_INT32, "Expected Files", njcr->ExpectedFiles, + OT_INT32, "Percent Complete", 100*(njcr->num_files_examined/njcr->ExpectedFiles), + OT_END); + } + + sendit(p, strlen(p), sp); + ow.get_output(OT_CLEAR, OT_END); + + if (njcr->JobFiles > 0) { + njcr->lock(); + ow.get_output(OT_STRING, "Processing file", njcr->last_fname, OT_END); + njcr->unlock(); + } + + if (njcr->store_bsock) { + ow.get_output(OT_INT64, "SDReadSeqNo", (int64_t)njcr->store_bsock->read_seqno, + OT_INT, "fd", njcr->store_bsock->m_fd, + OT_INT, "SDtls", (njcr->store_bsock->tls)?1:0, + OT_END); + } else { + ow.get_output(OT_STRING, "SDSocket", "closed", OT_END); + } + ow.get_output(OT_END_OBJ, OT_END); + sendit(p, strlen(p), sp); + } + endeach_jcr(njcr); +} + static void list_running_jobs(STATUS_PKT *sp) { - list_running_jobs_plain(sp); + if (sp->api) { + list_running_jobs_api(sp); + } else { + list_running_jobs_plain(sp); + } } /* diff --git a/bacula/src/filed/verify_vol.c b/bacula/src/filed/verify_vol.c index 69ce3d7118..5fba40c38b 100644 --- a/bacula/src/filed/verify_vol.c +++ b/bacula/src/filed/verify_vol.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -22,6 +22,7 @@ * * Kern Sibbald, July MMII * + * Data verification added by Eric Bollengier */ #include "bacula.h" @@ -82,6 +83,9 @@ public: /* Scan the fileset to know if we want to check checksums or st_size */ void scan_fileset(); + /* Check the catalog to locate the file */ + void check_accurate(); + /* In cleanup, we reset the current file size to -1 */ void reset_size() { size = -1; @@ -184,6 +188,14 @@ void v_ctx::skip_sparse_header(char **data, uint32_t *length) *length -= OFFSET_FADDR_SIZE; } +void v_ctx::check_accurate() +{ + attr->fname = jcr->last_fname; /* struct stat is still valid, but not the fname */ + if (accurate_check_file(jcr, attr, digest)) { + jcr->setJobStatus(JS_Differences); + } +} + /* * If extracting, close any previous stream */ @@ -199,7 +211,7 @@ bool v_ctx::close_previous_stream() /* Check the size if possible */ if (check_size && size >= 0) { - if (attr->type == FT_REG && size != attr->statp.st_size) { + if (attr->type == FT_REG && size != (int64_t)attr->statp.st_size) { Dmsg1(50, "Size comparison failed for %s\n", jcr->last_fname); Jmsg(jcr, M_INFO, 0, _(" st_size differs on \"%s\". Vol: %s File: %s\n"), @@ -238,11 +250,11 @@ void do_verify_volume(JCR *jcr) uint32_t VolSessionId, VolSessionTime, file_index; char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)]; int stat; + int bget_ret = 0; char *wbuf; /* write buffer */ uint32_t wsize; /* write size */ uint32_t rsize; /* read size */ - bool msg_encrypt = false; - bool do_chksum; + bool msg_encrypt = false, do_check_accurate=false; v_ctx vctx(jcr); ATTR *attr = vctx.attr; @@ -277,14 +289,16 @@ void do_verify_volume(JCR *jcr) jcr->compress_buf_size = compress_buf_size; } - GetMsg *fdmsg = New(GetMsg(jcr, sd, rec_header, GETMSG_MAX_MSG_SIZE)); + GetMsg *fdmsg; + fdmsg = New(GetMsg(jcr, sd, rec_header, GETMSG_MAX_MSG_SIZE)); + fdmsg->start_read_sock(); - bmessage *bmsg = New(bmessage(GETMSG_MAX_MSG_SIZE)); + bmessage *bmsg = fdmsg->new_msg(); /* get a message, to exchange with fdmsg */ /* * Get a record from the Storage daemon */ - while (fdmsg->bget_msg(&bmsg) >= 0 && !job_canceled(jcr)) { + while ((bget_ret = fdmsg->bget_msg(&bmsg)) >= 0 && !job_canceled(jcr)) { /* Remember previous stream type */ vctx.prev_stream = vctx.stream; @@ -303,8 +317,12 @@ void do_verify_volume(JCR *jcr) /* * Now we expect the Stream Data */ - if (fdmsg->bget_msg(&bmsg) < 0) { - Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror()); + if ((bget_ret = fdmsg->bget_msg(&bmsg)) < 0) { + if (bget_ret != BNET_EXT_TERMINATE) { + Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror()); + } else { + /* The error has been handled somewhere else, just quit */ + } goto bail_out; } if (size != ((uint32_t)bmsg->origlen)) { @@ -321,6 +339,12 @@ void do_verify_volume(JCR *jcr) if (!vctx.close_previous_stream()) { goto bail_out; } + if (do_check_accurate) { + vctx.check_accurate(); + } + /* Next loop, we want to check the file (or we do it with the md5) */ + do_check_accurate = true; + /* * Unpack attributes and do sanity check them */ @@ -421,7 +445,6 @@ void do_verify_volume(JCR *jcr) digest, digest_code, jcr->JobFiles); } else if (jcr->getJobLevel() == L_VERIFY_DATA) { - /* Compare digest */ if (vctx.check_chksum && *digest) { /* probably an empty file, we can create an empty crypto session */ @@ -440,6 +463,10 @@ void do_verify_volume(JCR *jcr) Dmsg3(50, "Signature verification failed for %s %s != %s\n", jcr->last_fname, digest, vctx.digest); } + if (do_check_accurate) { + vctx.check_accurate(); + do_check_accurate = false; /* Don't do it in the next loop */ + } } /* Compute size and checksum for level=Data */ @@ -467,8 +494,7 @@ void do_verify_volume(JCR *jcr) case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: - do_chksum=true; - if (!(attr->type == FT_RAW || attr->type == FT_FIFO || attr->type == FT_REG)) { + if (!(attr->type == FT_RAW || attr->type == FT_FIFO || attr->type == FT_REG || attr->type == FT_REGE)) { break; } @@ -483,19 +509,6 @@ void do_verify_volume(JCR *jcr) vctx.skip_sparse_header(&wbuf, &wsize); } - /* On Windows, the checksum is computed after the compression - * On Unix, the checksum is computed before the compression - */ - if (vctx.stream == STREAM_WIN32_GZIP_DATA - || vctx.stream == STREAM_WIN32_DATA - || vctx.stream == STREAM_WIN32_COMPRESSED_DATA - || vctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA - || vctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) - { - do_chksum = false; - vctx.update_checksum(wbuf, wsize); - } - if (vctx.stream == STREAM_GZIP_DATA || vctx.stream == STREAM_SPARSE_GZIP_DATA || vctx.stream == STREAM_WIN32_GZIP_DATA @@ -513,10 +526,7 @@ void do_verify_volume(JCR *jcr) } } - /* Unix way to deal with checksums */ - if (do_chksum) { - vctx.update_checksum(wbuf, wsize); - } + vctx.update_checksum(wbuf, wsize); if (vctx.stream == STREAM_WIN32_GZIP_DATA || vctx.stream == STREAM_WIN32_DATA @@ -541,9 +551,19 @@ void do_verify_volume(JCR *jcr) } } /* end switch */ } /* end while bnet_get */ + if (bget_ret == BNET_EXT_TERMINATE) { + goto bail_out; + } if (!vctx.close_previous_stream()) { goto bail_out; } + /* Check the last file */ + if (do_check_accurate) { + vctx.check_accurate(); + } + if (!accurate_finish(jcr)) { + goto bail_out; + } jcr->setJobStatus(JS_Terminated); goto ok_out; @@ -551,8 +571,7 @@ bail_out: jcr->setJobStatus(JS_ErrorTerminated); ok_out: - Dmsg0(215, "wait BufferedMsg\n"); - fdmsg->wait_read_sock(); + fdmsg->wait_read_sock(jcr->is_job_canceled()); delete bmsg; free_GetMsg(fdmsg); if (jcr->compress_buf) { diff --git a/bacula/src/filed/xattr.h b/bacula/src/filed/xattr.h new file mode 100644 index 0000000000..1c406189f5 --- /dev/null +++ b/bacula/src/filed/xattr.h @@ -0,0 +1,76 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef __BXATTR_H_ +#define __BXATTR_H_ + +#if defined(HAVE_LINUX_OS) +#define BXATTR_ENOTSUP EOPNOTSUPP +#elif defined(HAVE_DARWIN_OS) +#define BXATTR_ENOTSUP ENOTSUP +#elif defined(HAVE_HURD_OS) +#define BXATTR_ENOTSUP ENOTSUP +#endif + +/* + * Magic used in the magic field of the xattr struct. + * This way we can see we encounter a valid xattr struct. + */ +#define XATTR_MAGIC 0x5C5884 + +/* + * Internal representation of an extended attribute. + */ +struct xattr_t { + uint32_t magic; + uint32_t name_length; + char *name; + uint32_t value_length; + char *value; +}; + +/* + * Internal representation of an extended attribute hardlinked file. + */ +struct xattr_link_cache_entry_t { + uint32_t inum; + char *target; +}; + +#define BXATTR_FLAG_SAVE_NATIVE 0x01 +#define BXATTR_FLAG_RESTORE_NATIVE 0x02 + +/* + * Internal tracking data. + */ +struct xattr_ctx_t { + uint32_t flags; /* See BXATTR_FLAG_* */ + uint32_t current_dev; + uint32_t nr_errors; + uint32_t nr_saved; + POOLMEM *content; + uint32_t content_length; + alist *link_cache; +}; + +#define MAX_XATTR_LENGTH (1 * 1024 * 1024) /* 1 Mb */ + +#define XATTR_BUFSIZ 1024 + +#endif /* __BXATTR_H_ */ diff --git a/bacula/src/fileopts.h b/bacula/src/fileopts.h index 0dee79272f..76fbd9eddf 100644 --- a/bacula/src/fileopts.h +++ b/bacula/src/fileopts.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2001-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -23,7 +22,6 @@ * Kern Sibbald MMI * * Extracted from findlib/find.h Nov 2010 - * */ #ifndef __BFILEOPTS_H diff --git a/bacula/src/filetypes.h b/bacula/src/filetypes.h index 96ef11ed37..82f08f3c54 100644 --- a/bacula/src/filetypes.h +++ b/bacula/src/filetypes.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Stream definitions. Split from baconfig.h Nov 2010 * * Kern Sibbald, MM - * */ #ifndef __BFILETYPES_H diff --git a/bacula/src/findlib/attribs.c b/bacula/src/findlib/attribs.c index 9d249b1c88..651547b306 100644 --- a/bacula/src/findlib/attribs.c +++ b/bacula/src/findlib/attribs.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -354,7 +354,6 @@ void encode_stat(char *buf, struct stat *statp, int stat_size, int32_t LinkFI, i return; } - /* Do casting according to unknown type to keep compiler happy */ #ifdef HAVE_TYPEOF #define plug(st, val) st = (typeof st)val @@ -542,7 +541,7 @@ bool set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd) bool ok = true; boffset_t fsize; - if (uid_set) { + if (!uid_set) { my_uid = getuid(); my_gid = getgid(); uid_set = true; @@ -574,6 +573,7 @@ bool set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd) * try to do a chmod as that will update the file behind it. */ if (attr->type == FT_LNK) { +#ifdef HAVE_LCHOWN /* Change owner of link, not of real file */ if (lchown(attr->ofname, attr->statp.st_uid, attr->statp.st_gid) < 0 && print_error) { berrno be; @@ -581,6 +581,20 @@ bool set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd) attr->ofname, be.bstrerror()); ok = false; } +#endif +#ifdef HAVE_LUTIMES + struct timeval times[2]; + times[0].tv_sec = attr->statp.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = attr->statp.st_mtime; + times[1].tv_usec = 0; + if (lutimes(attr->ofname, times) < 0 && print_error) { + berrno be; + Jmsg2(jcr, M_ERROR, 0, _("Unable to set file times %s: ERR=%s\n"), + attr->ofname, be.bstrerror()); + ok = false; + } +#endif } else { /* * At this point, we have a file that is not a LINK diff --git a/bacula/src/findlib/bfile.c b/bacula/src/findlib/bfile.c index d86e666206..067677dc45 100644 --- a/bacula/src/findlib/bfile.c +++ b/bacula/src/findlib/bfile.c @@ -506,7 +506,7 @@ int bopen(BFILE *bfd, const char *fname, uint64_t flags, mode_t mode) Dmsg1(dbglvl, "open file %s\n", fname); /* We use fnctl to set O_NOATIME if requested to avoid open error */ - bfd->fid = open(fname, flags & ~O_NOATIME, mode); + bfd->fid = open(fname, (flags | O_CLOEXEC) & ~O_NOATIME, mode); /* Set O_NOATIME if possible */ if (bfd->fid != -1 && flags & O_NOATIME) { diff --git a/bacula/src/findlib/fstype.c b/bacula/src/findlib/fstype.c index ab31c2dac4..a801d9afaa 100644 --- a/bacula/src/findlib/fstype.c +++ b/bacula/src/findlib/fstype.c @@ -404,7 +404,7 @@ bool read_mtab(mtab_handler_t *mtab_handler, void *user_ctx) struct mnttab mnt; P(mutex); - if ((mntfp = fopen(MNTTAB, "r")) == NULL) { + if ((mntfp = bfopen(MNTTAB, "r")) == NULL) { V(mutex); return false; } diff --git a/bacula/src/findlib/protos.h b/bacula/src/findlib/protos.h index b0ee97247a..b6b8acff82 100644 --- a/bacula/src/findlib/protos.h +++ b/bacula/src/findlib/protos.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -22,6 +22,8 @@ */ /* from attribs.c */ +bool check_directory_acl(char **last_dir, alist *dir_acl, const char *path); + void encode_stat (char *buf, struct stat *statp, int stat_size, int32_t LinkFI, int data_stream); int decode_stat (char *buf, struct stat *statp, int stat_size, int32_t *LinkFI); int32_t decode_LinkFI (char *buf, struct stat *statp, int stat_size); diff --git a/bacula/src/host.h.in b/bacula/src/host.h.in index 821c7d128e..b110cdabc5 100644 --- a/bacula/src/host.h.in +++ b/bacula/src/host.h.in @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index c2b85f7ea5..a4d2fb070a 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -58,6 +58,9 @@ #define JT_MIGRATE 'g' /* Migration Job */ #define JT_SCAN 'S' /* Scan Job */ +/* Used to handle ClientAcl in various commands, not stored in the DB */ +#define JT_BACKUP_RESTORE '*' /* Backup or Restore Job */ + /* Job Status. Some of these are stored in the DB */ #define JS_Canceled 'A' /* canceled by user */ #define JS_Blocked 'B' /* blocked */ @@ -86,9 +89,10 @@ #define JS_WaitDevice 'q' /* Queued waiting for device */ #define JS_WaitStoreRes 's' /* Waiting for storage resource */ #define JS_WaitStartTime 't' /* Waiting for start time */ +#define JS_CloudUpload 'u' /* Cloud upload */ +#define JS_CloudDownload 'w' /* Cloud download */ - -/* Migration selection types */ +/* Migration selection types. Do not change the order. */ enum { MT_SMALLEST_VOL = 1, MT_OLDEST_VOL, @@ -109,18 +113,20 @@ enum { #define job_waiting(jcr) \ (jcr->job_started && \ - (jcr->JobStatus == JS_WaitFD || \ - jcr->JobStatus == JS_WaitSD || \ - jcr->JobStatus == JS_WaitMedia || \ - jcr->JobStatus == JS_WaitMount || \ - jcr->JobStatus == JS_WaitStoreRes || \ - jcr->JobStatus == JS_WaitJobRes || \ - jcr->JobStatus == JS_WaitClientRes|| \ - jcr->JobStatus == JS_WaitMaxJobs || \ - jcr->JobStatus == JS_WaitPriority || \ - jcr->SDJobStatus == JS_WaitMedia || \ - jcr->SDJobStatus == JS_WaitMount || \ - jcr->SDJobStatus == JS_WaitDevice || \ + (jcr->JobStatus == JS_WaitFD || \ + jcr->JobStatus == JS_WaitSD || \ + jcr->JobStatus == JS_WaitMedia || \ + jcr->JobStatus == JS_WaitMount || \ + jcr->JobStatus == JS_WaitStoreRes || \ + jcr->JobStatus == JS_WaitJobRes || \ + jcr->JobStatus == JS_WaitClientRes || \ + jcr->JobStatus == JS_WaitMaxJobs || \ + jcr->JobStatus == JS_WaitPriority || \ + jcr->SDJobStatus == JS_WaitMedia || \ + jcr->SDJobStatus == JS_WaitMount || \ + jcr->SDJobStatus == JS_WaitDevice || \ + jcr->SDJobStatus == JS_CloudUpload || \ + jcr->SDJobStatus == JS_CloudDownload || \ jcr->SDJobStatus == JS_WaitMaxJobs)) @@ -145,6 +151,7 @@ struct bpContext; #ifdef FILE_DAEMON +class VSSClient; class htable; class XACL; class snapshot_manager; @@ -305,7 +312,10 @@ public: POOLMEM *comment; /* Comment for this Job */ int64_t max_bandwidth; /* Bandwidth limit for this Job */ htable *path_list; /* Directory list (used by findlib) */ - + int job_uid; /* UID used during job session */ + char *job_user; /* Specific permission for a job */ + char *job_group; /* Specific permission for a job */ + POOLMEM *StatusErrMsg; /* Error message displayed in the job report */ uint32_t getErrors() { return JobErrors + SDErrors; }; /* Get error count */ /* Daemon specific part of JCR */ @@ -402,6 +412,7 @@ public: bool RescheduleIncompleteJobs; /* set if incomplete can be rescheduled */ bool use_all_JobIds; /* Use all jobids present in command line */ bool sd_client; /* This job runs as SD client */ + bool dummy_jobmedia; /* Dummy JobMedia written */ #endif /* DIRECTOR_DAEMON */ @@ -450,6 +461,7 @@ public: uint64_t base_size; /* compute space saved with base job */ utime_t snapshot_retention; /* Snapshot retention (from director) */ snapshot_manager *snap_mgr; /* Snapshot manager */ + VSSClient *pVSSClient; /* VSS handler */ #endif /* FILE_DAEMON */ @@ -496,6 +508,7 @@ public: bool Resched; /* Job may be rescheduled */ bool bscan_insert_jobmedia_records; /*Bscan: needs to insert job media records */ bool sd_client; /* Set if acting as client */ + bool use_new_match_all; /* TODO: Remove when the match_bsr() will be well tested */ /* Parmaters for Open Read Session */ BSR *bsr; /* Bootstrap record -- has everything */ diff --git a/bacula/src/lib/Makefile.in b/bacula/src/lib/Makefile.in index 4726e7734e..56b6d36fbd 100644 --- a/bacula/src/lib/Makefile.in +++ b/bacula/src/lib/Makefile.in @@ -1,5 +1,5 @@ # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @MCOMMON@ @@ -8,6 +8,10 @@ LIBBAC_LT_RELEASE = @LIBBAC_LT_RELEASE@ LIBBACCFG_LT_RELEASE = @LIBBACCFG_LT_RELEASE@ +ZLIBS = @ZLIBS@ +DEBUG = @DEBUG@ +CAP_LIBS = @CAP_LIBS@ + # srcdir = . VPATH = . @@ -20,10 +24,6 @@ topdir = ../.. # this dir relative to top dir thisdir = src/lib -DEBUG = @DEBUG@ -CAP_LIBS = @CAP_LIBS@ -ZLIBS = @ZLIBS@ - first_rule: all dummy: @@ -31,43 +31,42 @@ dummy: # include files installed when using libtool # INCLUDE_FILES = ../baconfig.h ../bacula.h ../bc_types.h \ - ../config.h ../jcr.h ../version.h \ - address_conf.h alist.h attr.h base64.h \ - berrno.h bits.h bpipe.h breg.h bregex.h \ - bsock.h btime.h btimers.h crypto.h dlist.h \ - fnmatch.h guid_to_name.h htable.h lex.h \ - lib.h md5.h mem_pool.h message.h \ - openssl.h plugins.h protos.h queue.h rblist.h \ - runscript.h rwlock.h serial.h sellist.h sha1.h \ - smartall.h status.h tls.h tree.h var.h \ - waitq.h watchdog.h workq.h \ - parse_conf.h ini.h \ - lockmgr.h devlock.h + ../config.h ../jcr.h ../version.h \ + address_conf.h alist.h attr.h base64.h \ + berrno.h bits.h bjson.h bpipe.h breg.h bregex.h \ + bsock.h btime.h btimers.h crypto.h dlist.h \ + flist.h fnmatch.h guid_to_name.h htable.h lex.h \ + lib.h lz4.h md5.h mem_pool.h message.h \ + openssl.h parse_conf.h plugins.h protos.h queue.h rblist.h \ + runscript.h rwlock.h serial.h sellist.h sha1.h sha2.h \ + smartall.h status.h tls.h tree.h var.h \ + watchdog.h workq.h ini.h \ + lockmgr.h devlock.h output.h bwlimit.h # # libbac # LIBBAC_SRCS = attr.c base64.c berrno.c bsys.c binflate.c bget_msg.c \ - crc32.c \ - bnet.c bnet_server.c bsock.c bpipe.c bsnprintf.c btime.c \ - cram-md5.c crypto.c daemon.c edit.c fnmatch.c \ - guid_to_name.c hmac.c jcr.c lex.c alist.c dlist.c \ - md5.c message.c mem_pool.c openssl.c \ - plugins.c priv.c queue.c bregex.c \ - runscript.c rwlock.c scan.c sellist.c serial.c sha1.c \ - signal.c smartall.c rblist.c tls.c tree.c \ - util.c var.c watchdog.c workq.c btimers.c \ - address_conf.c breg.c htable.c lockmgr.c devlock.c - -LIBBAC_OBJS = $(LIBBAC_SRCS:.c=.o) -LIBBAC_LOBJS = $(LIBBAC_SRCS:.c=.lo) + bnet.c bnet_server.c bsock.c bpipe.c bsnprintf.c btime.c \ + cram-md5.c crc32.c crypto.c daemon.c edit.c fnmatch.c \ + guid_to_name.c hmac.c jcr.c lex.c lz4.c alist.c dlist.c \ + md5.c message.c mem_pool.c openssl.c \ + plugins.c priv.c queue.c bregex.c \ + runscript.c rwlock.c scan.c sellist.c serial.c sha1.c sha2.c \ + signal.c smartall.c rblist.c tls.c tree.c \ + util.c var.c watchdog.c workq.c btimers.c \ + worker.c flist.c \ + address_conf.c breg.c htable.c lockmgr.c devlock.c output.c bwlimit.c + +LIBBAC_OBJS = $(LIBBAC_SRCS:.c=.o) +LIBBAC_LOBJS = $(LIBBAC_SRCS:.c=.lo) # # libbaccfg (config functions) # -LIBBACCFG_SRCS = ini.c parse_conf.c res.c -LIBBACCFG_OBJS = $(LIBBACCFG_SRCS:.c=.o) -LIBBACCFG_LOBJS = $(LIBBACCFG_SRCS:.c=.lo) +LIBBACCFG_SRCS = ini.c parse_conf.c res.c bjson.c +LIBBACCFG_OBJS = $(LIBBACCFG_SRCS:.c=.o) +LIBBACCFG_LOBJS = $(LIBBACCFG_SRCS:.c=.lo) .SUFFIXES: .c .cc .o .lo .ch .dvi .pdf .tex .view .w .1 .PHONY: @@ -102,9 +101,9 @@ libbac.a: $(LIBBAC_OBJS) $(AR) rc $@ $(LIBBAC_OBJS) $(RANLIB) $@ -libbac.la: Makefile $(LIBBAC_LOBJS) +libbac.la: Makefile $(LIBBAC_LOBJS) @echo "Making $@ ..." - $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -o $@ $(LIBBAC_LOBJS) -export-dynamic -rpath $(libdir) -release $(LIBBAC_LT_RELEASE) $(WRAPLIBS) $(CAP_LIBS) $(ZLIBS) $(OPENSSL_LIBS) $(LIBS) $(DLLIBS) + $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -o $@ $(LIBBAC_LOBJS) -export-dynamic -rpath $(libdir) -release $(LIBBAC_LT_RELEASE) $(WRAPLIBS) $(CAP_LIBS) $(ZLIBS) $(OPENSSL_LIBS) $(LIBS) $(DLLIBS) libbaccfg.a: $(LIBBACCFG_OBJS) @echo "Making $@ ..." @@ -119,6 +118,20 @@ Makefile: $(srcdir)/Makefile.in $(topdir)/config.status cd $(topdir) \ && CONFIG_FILES=$(thisdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status +wait_test: Makefile bsys.c + $(RMF) bsys.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) bsys.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -L../findlib -o $@ bsys.o $(DLIB) -lbac -lbacfind -lm $(LIBS) $(OPENSSL_LIBS) + rm -f bsys.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) bsys.c + +output_test: Makefile output.c + $(RMF) output.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) output.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ output.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + rm -f output.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) output.c + lockmgr_test: Makefile lockmgr.c $(RMF) lockmgr.o $(CXX) -D _TEST_IT $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) lockmgr.c @@ -133,6 +146,27 @@ base64_test: Makefile base64.c rm -f base64.o $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) base64.c +flist_test: Makefile flist.c + $(RMF) flist.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) flist.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ flist.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + $(RMF) flist.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) flist.c + +worker_test: Makefile worker.c + $(RMF) worker.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) worker.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ worker.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + $(RMF) worker.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) worker.c + +workq_test: Makefile workq.c + $(RMF) workq.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) workq.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ workq.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + $(RMF) workq.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) workq.c + rwlock_test: Makefile rwlock.c $(RMF) rwlock.o $(CXX) -DTEST_RWLOCK $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) rwlock.c @@ -147,12 +181,20 @@ devlock_test: Makefile $(RMF) devlock.o $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) devlock.c -htable_test: Makefile +htable_test: Makefile $(RMF) htable.o $(CXX) -DTEST_SMALL_HTABLE -DTEST_NON_CHAR -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) htable.c - $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ htable.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ htable.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) $(RMF) htable.o - $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) htable.c + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) htable.c + +alist_test: Makefile alist.c + $(RMF) alist.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) alist.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -o $@ alist.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + $(RMF) alist.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) alist.c + sellist_test: Makefile sellist.c $(RMF) sellist.o diff --git a/bacula/src/lib/alist.c b/bacula/src/lib/alist.c index ff88cf5866..d947c56318 100644 --- a/bacula/src/lib/alist.c +++ b/bacula/src/lib/alist.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -23,6 +23,12 @@ * it simply malloc's a bigger array controlled by num_grow. * Default is to realloc the pointer array for each new member. * + * Note: the list can have holes (empty items). This is done by + * using get() and put(). If you are using this kind of indexed + * list, you cannot use: prepend() and remove() as they will + * reorder the list. So, in the ilist array, these functions are + * disabled and the put method is defined. + * * Kern Sibbald, June MMIII * */ @@ -33,17 +39,31 @@ * Private grow list function. Used to insure that * at least one more "slot" is available. */ -void alist::grow_list() +void baselist::grow_list() { + int i; + int new_max_items; + + /* put() can insert and item anywhere in the list so + it's important to allocate at least last_item+1 items */ + int min_grow = MAX(10, last_item+1); + if (num_grow < min_grow) { + num_grow = min_grow; /* default if not initialized */ + } + if (items == NULL) { - if (num_grow == 0) { - num_grow = 1; /* default if not initialized */ - } items = (void **)malloc(num_grow * sizeof(void *)); + for (i=0; i= max_items) { + new_max_items = last_item + num_grow; + items = (void **)realloc(items, new_max_items * sizeof(void *)); + for (i=max_items; i= num_items) { + if (cur_item >= last_item) { return NULL; } else { return items[cur_item++]; @@ -88,67 +108,104 @@ void *alist::prev() /* * prepend an item to the list -- i.e. add to beginning */ -void alist::prepend(void *item) { +void alist::prepend(void *item) +{ grow_list(); if (num_items == 0) { items[num_items++] = item; + if (num_items > last_item) { + last_item = num_items; + } return; } - for (int i=num_items; i > 0; i--) { + for (int i=last_item; i > 0; i--) { items[i] = items[i-1]; } items[0] = item; num_items++; + last_item++; } /* * Append an item to the list */ -void alist::append(void *item) { +void baselist::append(void *item) +{ grow_list(); - items[num_items++] = item; + items[last_item++] = item; + num_items++; } -/* Remove an item from the list */ -void * alist::remove(int index) +/* + * Put an item at a particular index + */ +void ilist::put(int index, void *item) +{ + if (index > last_item) { + last_item = index; + } + grow_list(); + if (items[index] == NULL) { + num_items++; + } + items[index] = item; +} + + +/* + * Remove an item from the list + * Note: you must free the item when + * you are done with it. + */ +void * baselist::remove_item(int index) { void *item; - if (index < 0 || index >= num_items) { + if (index < 0 || index >= last_item) { return NULL; } item = items[index]; - num_items--; - for (int i=index; i < num_items; i++) { + + /* last_item is from 1..n, we work from 0..n-1 */ + for (int i=index; i < (last_item-1); i++) { items[i] = items[i+1]; } + + items[last_item-1] = NULL; /* The last item is shifted by one, the last slot is always free */ + + last_item--; /* We have shifted all items by 1 */ + num_items--; /* We have 1 item less */ + return item; } /* Get the index item -- we should probably allow real indexing here */ -void * alist::get(int index) +void * baselist::get(int index) { - if (index < 0 || index >= num_items) { + if (items == NULL || index < 0 || index > last_item) { return NULL; } return items[index]; } /* Destroy the list and its contents */ -void alist::destroy() +void baselist::destroy() { if (items) { if (own_items) { - for (int i=0; imylist.init(); printf("Manual allocation/destruction of list:\n"); @@ -183,7 +241,7 @@ int main() free(fileset); printf("Allocation/destruction using new delete\n"); - mlist = new alist(10); + mlist = New(alist(50)); for (int i=0; i<20; i++) { sprintf(buf, "This is item %d", i); @@ -192,11 +250,181 @@ int main() for (int i=0; i< mlist->size(); i++) { printf("Item %d = %s\n", i, (char *)mlist->get(i)); } + printf("\nIndexed test. Insert 210 items.\n"); + /* Test indexed list */ + mlist->destroy(); + delete mlist; + + { + printf("Test remove()\n"); + char *elt; + int i=0; + alist *alst = New(alist(10, owned_by_alist)); + alst->append(bstrdup("trash")); + alst->append(bstrdup("0")); + alst->append(bstrdup("1")); + alst->append(bstrdup("2")); + alst->append(bstrdup("3")); + free(alst->remove(0)); + foreach_alist(elt, alst) { + int nb = atoi(elt); + ASSERT(nb == i); + printf("%d %s\n", i++, elt); + } + delete alst; + } + + { + char *elt; + int i=0; + alist *alst = New(alist(10, owned_by_alist)); + alst->append(bstrdup("0")); + alst->append(bstrdup("1")); + alst->append(bstrdup("2")); + alst->append(bstrdup("trash")); + alst->append(bstrdup("3")); + free(alst->remove(3)); + foreach_alist(elt, alst) { + int nb = atoi(elt); + ASSERT(nb == i); + printf("%d %s\n", i++, elt); + } + delete alst; + } + + { + char *elt; + int i=0; + alist *alst = New(alist(10, owned_by_alist)); + alst->append(bstrdup("0")); + alst->append(bstrdup("1")); + alst->append(bstrdup("2")); + alst->append(bstrdup("3")); + alst->append(bstrdup("trash")); + free(alst->remove(4)); + foreach_alist(elt, alst) { + int nb = atoi(elt); + ASSERT(nb == i); + printf("%d %s\n", i++, elt); + } + delete alst; + } + + { + char *elt; + int i=0; + alist *alst = New(alist(10, owned_by_alist)); + alst->append(bstrdup("0")); + alst->append(bstrdup("1")); + alst->append(bstrdup("2")); + alst->append(bstrdup("3")); + alst->append(bstrdup("4")); + ASSERT(alst->remove(5) == NULL); + foreach_alist(elt, alst) { + int nb = atoi(elt); + ASSERT(nb == i); + printf("%d %s\n", i++, elt); + } + delete alst; + } + + { + printf("Test pop()\n"); + char *elt; + int i=0; + alist *alst = New(alist(10, owned_by_alist)); + alst->append(bstrdup("0")); + alst->append(bstrdup("1")); + alst->append(bstrdup("2")); + alst->append(bstrdup("3")); + alst->append(bstrdup("trash")); + ASSERT(alst->last_index() == 5); + free(alst->pop()); + ASSERT(alst->last_index() == 4); + foreach_alist(elt, alst) { + int nb = atoi(elt); + ASSERT(nb == i); + printf("%d %s\n", i++, elt); + } + delete alst; + } + { + ilist *ilst = New(ilist(10, owned_by_alist)); + sprintf(buf, "This is item 10"); + ilst->put(10, bstrdup(buf)); + printf("ilst size is %d. last_item=%d. max_items=%d\n", + ilst->size(), ilst->last_index(), ilst->max_size()); + ASSERT(ilst->size() == 1); + delete ilst; + } + + { + ilist *ilst = New(ilist(10, not_owned_by_alist)); + ilst->put(15, (char *)"something"); + printf("ilst size is %d. last_item=%d. max_items=%d\n", + ilst->size(), ilst->last_index(), ilst->max_size()); + ASSERT(ilst->size() == 1); + ASSERT(ilst->last_index() == 15); + delete ilst; + } + + ilist *ilst = New(ilist(50)); + for (int i=0; i<115; i++) { + sprintf(buf, "This is item %d", i); + ilst->put(i, bstrdup(buf)); + printf("%s\n", buf); + } + printf("ilst size is %d. last_item=%d. max_items=%d\n", + ilst->size(), ilst->last_index(), ilst->max_size()); + for (int i=0; i< ilst->size(); i++) { + printf("Item %d = %s\n", i, (char *)ilst->get(i)); + } + + delete ilst; + + printf("Test alist push().\n"); + mlist = New(alist(10)); + + printf("mlist size is %d. last_item=%d. max_items=%d\n", + mlist->size(), mlist->last_index(), mlist->max_size()); + for (int i=0; i<20; i++) { + sprintf(buf, "This is item %d", i); + mlist->push(bstrdup(buf)); + printf("mlist size is %d. last_item=%d. max_items=%d\n", + mlist->size(), mlist->last_index(), mlist->max_size()); + } + printf("Test alist pop()\n"); + for (int i=0; (bp=(char *)mlist->pop()); i++) { + printf("Item %d = %s\n", i, bp); + free(bp); + } + printf("mlist size is %d. last_item=%d. max_items=%d\n", + mlist->size(), mlist->last_index(), mlist->max_size()); + + for (int i=0; imax_size(); i++) { + bp = (char *) mlist->get(i); + if (bp != NULL) { + printf("Big problem. Item %d item=%p, but should be NULL\n", i, bp); + } + } + printf("Done push() pop() tests\n"); delete mlist; + ilst = New(ilist(10, not_owned_by_alist)); + ilst->put(1, ilst); + ilst->append((void*)1); + //ilist->first(); + //ilist->remove(1); + delete ilst; + { + ilist a(4, not_owned_by_alist); + a.append((void*)"test1"); + ilist b(4, not_owned_by_alist); + bmemzero(&b, sizeof b); + b.append((void*)"test1"); + } sm_dump(false); /* test program */ - } #endif diff --git a/bacula/src/lib/alist.h b/bacula/src/lib/alist.h index 4d83c49d31..5308bd9792 100644 --- a/bacula/src/lib/alist.h +++ b/bacula/src/lib/alist.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,6 +21,8 @@ */ +extern bool is_null(const void *ptr); + /* * There is a lot of extra casting here to work around the fact * that some compilers (Sun and Visual C++) do not accept @@ -57,29 +59,29 @@ enum { /* * Array list -- much like a simplified STL vector - * array of pointers to inserted items + * array of pointers to inserted items. baselist is + * the common code between alist and ilist */ -class alist : public SMARTALLOC { - void **items; - int num_items; - int max_items; +class baselist : public SMARTALLOC { +protected: + void **items; /* from 0..n-1 */ + int num_items; /* from 1..n */ + int last_item; /* maximum item index (1..n) */ + int max_items; /* maximum possible items (array size) (1..n) */ int num_grow; - int cur_item; + int cur_item; /* from 1..n */ bool own_items; void grow_list(void); + void *remove_item(int index); public: - alist(int num = 10, bool own=true); - ~alist(); - void init(int num = 10, bool own=true); + baselist(int num = 100, bool own=true); + ~baselist(); + void init(int num = 100, bool own=true); void append(void *item); - void prepend(void *item); - void *remove(int index); void *get(int index); bool empty() const; - void *prev(); - void *next(); - void *first(); - void *last(); + int last_index() const { return last_item; }; + int max_size() const { return max_items; }; void * operator [](int index) const; int current() const { return cur_item; }; int size() const; @@ -88,20 +90,43 @@ public: /* Use it as a stack, pushing and poping from the end */ void push(void *item) { append(item); }; - void *pop() { return remove(num_items-1); }; + void *pop() { return remove_item(num_items-1); }; +}; + +class alist: public baselist +{ +public: + alist(int num = 100, bool own=true): baselist(num, own) {}; + void *prev(); + void *next(); + void *last(); + void *first(); + void prepend(void *item); + void *remove(int index) { return remove_item(index);}; +}; + +/* + * Indexed list -- much like a simplified STL vector + * array of pointers to inserted items + */ +class ilist : public baselist { +public: + ilist(int num = 100, bool own=true): baselist(num, own) {}; + /* put() is not compatible with remove(), prepend() or foreach_alist */ + void put(int index, void *item); }; /* * Define index operator [] */ -inline void * alist::operator [](int index) const { - if (index < 0 || index >= num_items) { +inline void * baselist::operator [](int index) const { + if (index < 0 || index >= max_items) { return NULL; } return items[index]; } -inline bool alist::empty() const +inline bool baselist::empty() const { return num_items == 0; } @@ -111,42 +136,37 @@ inline bool alist::empty() const * allowing us to mix C++ classes inside malloc'ed * C structures. Define before called in constructor. */ -inline void alist::init(int num, bool own) +inline void baselist::init(int num, bool own) { items = NULL; num_items = 0; + last_item = 0; max_items = 0; num_grow = num; own_items = own; } /* Constructor */ -inline alist::alist(int num, bool own) +inline baselist::baselist(int num, bool own) { init(num, own); } /* Destructor */ -inline alist::~alist() +inline baselist::~baselist() { destroy(); } - - /* Current size of list */ -inline int alist::size() const +inline int baselist::size() const { - /* - * Check for null pointer, which allows test - * on size to succeed even if nothing put in - * alist. - */ + if (is_null(this)) return 0; return num_items; } /* How much to grow by each time */ -inline void alist::grow(int num) +inline void baselist::grow(int num) { num_grow = num; } diff --git a/bacula/src/lib/attr.c b/bacula/src/lib/attr.c index 24dc05f455..4d5eddc424 100644 --- a/bacula/src/lib/attr.c +++ b/bacula/src/lib/attr.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -245,9 +245,9 @@ extern char *getuser(uid_t uid, char *name, int len); extern char *getgroup(gid_t gid, char *name, int len); /* - * Print an ls style message, also send M_RESTORED + * Print an ls style message, also send M_RESTORED/M_SAVED */ -void print_ls_output(JCR *jcr, ATTR *attr) +void print_ls_output(JCR *jcr, ATTR *attr, int message_type /* M_RESTORED */) { char buf[5000]; char ec1[30]; @@ -255,11 +255,16 @@ void print_ls_output(JCR *jcr, ATTR *attr) char *p, *f; guid_list *guid; + /* No need to compute everything if it's not required */ + if (!chk_dbglvl(dbglvl) && !is_message_type_set(jcr, message_type)) { + return; + } + if (attr->type == FT_DELETED) { /* TODO: change this to get last seen values */ bsnprintf(buf, sizeof(buf), "---------- - - - - ---------- -------- %s\n", attr->ofname); Dmsg1(dbglvl, "%s", buf); - Jmsg(jcr, M_RESTORED, 1, "%s", buf); + Jmsg(jcr, message_type, 1, "%s", buf); return; } @@ -292,5 +297,5 @@ void print_ls_output(JCR *jcr, ATTR *attr) *p++ = '\n'; *p = 0; Dmsg1(dbglvl, "%s", buf); - Jmsg(jcr, M_RESTORED, 1, "%s", buf); + Jmsg(jcr, message_type, 1, "%s", buf); } diff --git a/bacula/src/lib/bget_msg.c b/bacula/src/lib/bget_msg.c index fc20f9412b..753851647b 100644 --- a/bacula/src/lib/bget_msg.c +++ b/bacula/src/lib/bget_msg.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -55,6 +55,9 @@ int bget_msg(BSOCK *sock) if (sock->is_stop()) { /* error return */ return n; } + if (n == BNET_COMMAND) { + return n; + } /* BNET_SIGNAL (-1) return from bnet_recv() => network signal */ switch (sock->msglen) { @@ -160,5 +163,6 @@ int GetMsg::bget_msg(bmessage **pbmsg) msglen = bmsg->msglen; msg = bmsg->msg; + m_is_stop = bsock->is_stop() || bsock->is_error(); return bmsg->ret; } diff --git a/bacula/src/lib/bget_msg.h b/bacula/src/lib/bget_msg.h index 388b0e0630..e8b3458b5a 100644 --- a/bacula/src/lib/bget_msg.h +++ b/bacula/src/lib/bget_msg.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -84,7 +84,7 @@ public: virtual int bget_msg(bmessage **pbmsg=NULL); inline virtual void *do_read_sock_thread(void) { return NULL; }; inline virtual int start_read_sock() { return 0; }; - inline virtual void *wait_read_sock() { return NULL;}; + inline virtual void *wait_read_sock(int /*emergency_quit*/) { return NULL;}; virtual bool is_stop() { return (m_is_stop!=false); }; virtual bool is_done() { return (m_is_done!=false); }; diff --git a/bacula/src/lib/bjson.c b/bacula/src/lib/bjson.c new file mode 100644 index 0000000000..7ea1d36bf5 --- /dev/null +++ b/bacula/src/lib/bjson.c @@ -0,0 +1,408 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * Bacula Json library routines + * + * Kern Sibbald, September MMXII + * + */ + +#include "bacula.h" +#include "lib/breg.h" + +extern s_kw msg_types[]; +extern RES_TABLE resources[]; + +union URES { + MSGS res_msgs; + RES hdr; +}; + +#if defined(_MSC_VER) +extern "C" { // work around visual compiler mangling variables + extern URES res_all; +} +#else +extern URES res_all; +#endif + +struct display_filter +{ + /* default { { "Director": { "Name": aa, ...} }, { "Job": {..} */ + bool do_list; /* [ {}, {}, ..] or { "aa": {}, "bb": {}, ...} */ + bool do_one; /* { "Name": "aa", "Description": "test, ... } */ + bool do_only_data; /* [ {}, {}, {}, ] */ + char *resource_type; + char *resource_name; + regex_t directive_reg; +}; + +static void sendit(void *sock, const char *fmt, ...) +{ + char buf[3000]; + va_list arg_ptr; + + va_start(arg_ptr, fmt); + bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); + va_end(arg_ptr); + fputs(buf, stdout); + fflush(stdout); +} + +void init_hpkt(HPKT &hpkt) +{ + memset(&hpkt, 0, sizeof(hpkt)); + hpkt.edbuf = get_pool_memory(PM_EMSG); + hpkt.edbuf2 = get_pool_memory(PM_EMSG); + hpkt.json = true; + hpkt.hfunc = HF_DISPLAY; + hpkt.sendit = sendit; +} + +void term_hpkt(HPKT &hpkt) +{ + free_pool_memory(hpkt.edbuf); + free_pool_memory(hpkt.edbuf2); + memset(&hpkt, 0, sizeof(hpkt)); +} + +/* + * Strip long options out of fo->opts string so that + * they will not give us false matches for regular + * 1 or 2 character options. + */ +void strip_long_opts(char *out, const char *in) +{ + const char *p; + for (p=in; *p; p++) { + switch (*p) { + /* V, C, J, and P are long options, skip them */ + case 'V': + case 'C': + case 'J': + case 'P': + while (*p != ':') { + p++; /* skip to after : */ + } + break; + /* Copy everything else */ + default: + *out++ = *p; + break; + } + } + *out = 0; /* terminate string */ +} + +void edit_alist(HPKT &hpkt) +{ + bool f = true; + char *citem; + + pm_strcpy(hpkt.edbuf, " ["); + foreach_alist(citem, hpkt.list) { + if (!f) { + pm_strcat(hpkt.edbuf, ", "); + } + pm_strcat(hpkt.edbuf, quote_string(hpkt.edbuf2, citem)); + f = false; + } + pm_strcat(hpkt.edbuf, "]"); +} + +void edit_msg_types(HPKT &hpkt, DEST *dest) +{ + int i, j, count = 0; + bool first_type = true; + bool found; + + pm_strcpy(hpkt.edbuf, "["); + for (i=1; imsg_types)) { + found = false; + if (!first_type) pm_strcat(hpkt.edbuf, ","); + first_type = false; + for (j=0; msg_types[j].name; j++) { + if ((int)msg_types[j].token == i) { + pm_strcat(hpkt.edbuf, "\""); + pm_strcat(hpkt.edbuf, msg_types[j].name); + pm_strcat(hpkt.edbuf, "\""); + found = true; + break; + } + } + if (!found) { + sendit(NULL, "No find for type=%d\n", i); + } + count++; + } + } + /* + * Note, if we have more than half of the total items, + * redo using All and !item, which will give fewer items + * total. + */ + if (count > M_MAX/2) { + pm_strcpy(hpkt.edbuf, "[\"All\""); + for (i=1; imsg_types)) { + found = false; + pm_strcat(hpkt.edbuf, ","); + for (j=0; msg_types[j].name; j++) { + if ((int)msg_types[j].token == i) { + pm_strcat(hpkt.edbuf, "\"!"); + pm_strcat(hpkt.edbuf, msg_types[j].name); + pm_strcat(hpkt.edbuf, "\""); + found = true; + break; + } + } + if (!found) { + sendit(NULL, "No find for type=%d in second loop\n", i); + } + } else if (i == M_SAVED) { + /* Saved is not set by default, users must explicitly use it + * on the configuration line + */ + pm_strcat(hpkt.edbuf, ",\"Saved\""); + } + } + } + pm_strcat(hpkt.edbuf, "]"); +} + +bool display_global_item(HPKT &hpkt) +{ + bool found = true; + + if (hpkt.ritem->handler == store_res) { + display_res(hpkt); + } else if (hpkt.ritem->handler == store_str || + hpkt.ritem->handler == store_name || + hpkt.ritem->handler == store_password || + hpkt.ritem->handler == store_strname || + hpkt.ritem->handler == store_dir) { + display_string_pair(hpkt); + } else if (hpkt.ritem->handler == store_int32 || + hpkt.ritem->handler == store_pint32 || + hpkt.ritem->handler == store_size32) { + display_int32_pair(hpkt); + } else if (hpkt.ritem->handler == store_size64 || + hpkt.ritem->handler == store_int64 || + hpkt.ritem->handler == store_time || + hpkt.ritem->handler == store_speed) { + display_int64_pair(hpkt); + } else if (hpkt.ritem->handler == store_bool) { + display_bool_pair(hpkt); + } else if (hpkt.ritem->handler == store_msgs) { + display_msgs(hpkt); + } else if (hpkt.ritem->handler == store_bit) { + display_bit_pair(hpkt); + } else if (hpkt.ritem->handler == store_alist_res) { + found = display_alist_res(hpkt); /* In some cases, the list is null... */ + } else if (hpkt.ritem->handler == store_alist_str) { + found = display_alist_str(hpkt); /* In some cases, the list is null... */ + } else { + found = false; + } + + return found; +} + +/* + * Called here for each store_msgs resource + */ +void display_msgs(HPKT &hpkt) +{ + MSGS *msgs = (MSGS *)hpkt.ritem->value; /* Message res */ + DEST *dest; /* destination chain */ + int first = true; + + if (!hpkt.in_store_msg) { + hpkt.in_store_msg = true; + sendit(NULL, "\n \"Destinations\": ["); + } + for (dest=msgs->dest_chain; dest; dest=dest->next) { + if (dest->dest_code == hpkt.ritem->code) { + if (!first) sendit(NULL, ","); + first = false; + edit_msg_types(hpkt, dest); + switch (hpkt.ritem->code) { + /* Output only message types */ + case MD_STDOUT: + case MD_STDERR: + case MD_SYSLOG: + case MD_CONSOLE: + case MD_CATALOG: + sendit(NULL, "\n {\n \"Type\": \"%s\"," + "\n \"MsgTypes\": %s\n }", + hpkt.ritem->name, hpkt.edbuf); + break; + /* Output MsgTypes, Where */ + case MD_DIRECTOR: + case MD_FILE: + case MD_APPEND: + sendit(NULL, "\n {\n \"Type\": \"%s\"," + "\n \"MsgTypes\": %s,\n", + hpkt.ritem->name, hpkt.edbuf); + sendit(NULL, " \"Where\": [%s]\n }", + quote_where(hpkt.edbuf, dest->where)); + break; + /* Now we edit MsgTypes, Where, and Command */ + case MD_MAIL: + case MD_OPERATOR: + case MD_MAIL_ON_ERROR: + case MD_MAIL_ON_SUCCESS: + sendit(NULL, "\n {\n \"Type\": \"%s\"," + "\n \"MsgTypes\": %s,\n", + hpkt.ritem->name, hpkt.edbuf); + sendit(NULL, " \"Where\": [%s],\n", + quote_where(hpkt.edbuf, dest->where)); + sendit(NULL, " \"Command\": %s\n }", + quote_string(hpkt.edbuf, dest->mail_cmd)); + break; + } + } + } +} + +/* + * Called here if the ITEM_LAST is set in flags, + * that means there are no more items to examine + * for this resource and that we can close any + * open json list. + */ +void display_last(HPKT &hpkt) +{ + if (hpkt.in_store_msg) { + hpkt.in_store_msg = false; + sendit(NULL, "\n ]"); + } +} + +void display_alist(HPKT &hpkt) +{ + edit_alist(hpkt); + sendit(NULL, "%s", hpkt.edbuf); +} + +bool display_alist_str(HPKT &hpkt) +{ + hpkt.list = (alist *)(*(hpkt.ritem->value)); + if (!hpkt.list) { + return false; + } + sendit(NULL, "\n \"%s\":", hpkt.ritem->name); + display_alist(hpkt); + return true; +} + +bool display_alist_res(HPKT &hpkt) +{ + bool f = true; + alist *list; + RES *res; + + list = (alist *)(*(hpkt.ritem->value)); + if (!list) { + return false; + } + sendit(NULL, "\n \"%s\":", hpkt.ritem->name); + sendit(NULL, " ["); + foreach_alist(res, list) { + if (!f) { + sendit(NULL, ", "); + } + sendit(NULL, "%s", quote_string(hpkt.edbuf, res->name)); + f = false; + } + sendit(NULL, "]"); + return true; +} + +void display_res(HPKT &hpkt) +{ + RES *res; + + res = (RES *)*hpkt.ritem->value; + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, res->name)); +} + +void display_string_pair(HPKT &hpkt) +{ + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + quote_string(hpkt.edbuf, *hpkt.ritem->value)); +} + +void display_int32_pair(HPKT &hpkt) +{ + char ed1[50]; + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + edit_int64(*(int32_t *)hpkt.ritem->value, ed1)); +} + +void display_int64_pair(HPKT &hpkt) +{ + char ed1[50]; + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + edit_int64(*(int64_t *)hpkt.ritem->value, ed1)); +} + +void display_bool_pair(HPKT &hpkt) +{ + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + ((*(bool *)(hpkt.ritem->value)) == 0)?"false":"true"); +} + +void display_bit_pair(HPKT &hpkt) +{ + sendit(NULL, "\n \"%s\": %s", hpkt.ritem->name, + ((*(uint32_t *)(hpkt.ritem->value) & hpkt.ritem->code) + == 0)?"false":"true"); +} + +bool byte_is_set(char *byte, int num) +{ + int i; + bool found = false; + for (i=0; im_fd, &fdset); - tv.tv_sec = 1; - tv.tv_usec = 0; - select(bsock->m_fd + 1, NULL, &fdset, NULL, &tv); + fd_wait_data(bsock->m_fd, WAIT_WRITE, 1, 0); continue; } if (nwritten <= 0) { @@ -327,14 +320,6 @@ bool bnet_tls_client(TLS_CONTEXT *ctx, BSOCK * bsock, alist *verify_list) #define NO_DATA 4 /* Valid name, no data record of requested type. */ #endif -static IPADDR *add_any(int family) -{ - IPADDR *addr = New(IPADDR(family)); - addr->set_type(IPADDR::R_MULTIPLE); - addr->set_addr_any(); - return addr; -} - #if defined(HAVE_GETADDRINFO) /* * getaddrinfo.c - Simple example of using getaddrinfo(3) function. @@ -453,7 +438,7 @@ static const char *resolv_host(int family, const char *host, dlist * addr_list) #ifdef HAVE_IPV6 else { addr->set_addr6((struct in6_addr*)*p); - } + } #endif addr_list->append(addr); } @@ -461,8 +446,16 @@ static const char *resolv_host(int family, const char *host, dlist * addr_list) } return NULL; } -#endif - +#endif + +static IPADDR *add_any(int family) +{ + IPADDR *addr = New(IPADDR(family)); + addr->set_type(IPADDR::R_MULTIPLE); + addr->set_addr_any(); + return addr; +} + /* * i host = 0 means INADDR_ANY only for IPv4 */ @@ -551,6 +544,16 @@ const char *bnet_sig_to_ascii(int32_t msglen) return "BNET_SUB_PROMPT"; case BNET_TEXT_INPUT: return "BNET_TEXT_INPUT"; + case BNET_FDCALLED: + return "BNET_FDCALLED"; + case BNET_CMD_OK: + return "BNET_CMD_OK"; + case BNET_CMD_BEGIN: + return "BNET_CMD_BEGIN"; + case BNET_MAIN_PROMPT: + return "BNET_MAIN_PROMPT"; + case BNET_ERROR_MSG: + return "BNET_ERROR_MSG"; default: bsnprintf(buf, sizeof(buf), _("Unknown sig %d"), (int)msglen); return buf; @@ -558,25 +561,28 @@ const char *bnet_sig_to_ascii(int32_t msglen) } /* Initialize internal socket structure. - * This probably should be done in net_open + * This probably should be done in bsock.c */ -BSOCK *init_bsock(JCR * jcr, int sockfd, const char *who, const char *host, int port, - struct sockaddr *client_addr) +BSOCK *init_bsock(JCR *jcr, int sockfd, const char *who, + const char *host, int port, struct sockaddr *client_addr) { - Dmsg3(100, "who=%s host=%s port=%d\n", who, host, port); + Dmsg4(100, "socket=%d who=%s host=%s port=%d\n", sockfd, who, host, port); BSOCK *bsock = (BSOCK *)malloc(sizeof(BSOCK)); - memset(bsock, 0, sizeof(BSOCK)); + bmemzero(bsock, sizeof(BSOCK)); + bsock->m_master=bsock; /* don't use set_master() here */ bsock->m_fd = sockfd; bsock->tls = NULL; bsock->errors = 0; bsock->m_blocking = 1; bsock->pout_msg_no = &bsock->out_msg_no; + bsock->uninstall_send_hook_cb(); bsock->msg = get_pool_memory(PM_BSOCK); + bsock->cmsg = get_pool_memory(PM_BSOCK); bsock->errmsg = get_pool_memory(PM_MESSAGE); bsock->set_who(bstrdup(who)); bsock->set_host(bstrdup(host)); bsock->set_port(port); - memset(&bsock->peer_addr, 0, sizeof(bsock->peer_addr)); + bmemzero(&bsock->peer_addr, sizeof(bsock->peer_addr)); memcpy(&bsock->client_addr, client_addr, sizeof(bsock->client_addr)); bsock->timeout = BSOCK_TIMEOUT; bsock->set_jcr(jcr); @@ -589,6 +595,7 @@ BSOCK *dup_bsock(BSOCK *osock) osock->set_locking(); memcpy(bsock, osock, sizeof(BSOCK)); bsock->msg = get_pool_memory(PM_BSOCK); + bsock->cmsg = get_pool_memory(PM_BSOCK); bsock->errmsg = get_pool_memory(PM_MESSAGE); if (osock->who()) { bsock->set_who(bstrdup(osock->who())); @@ -600,6 +607,7 @@ BSOCK *dup_bsock(BSOCK *osock) bsock->src_addr = New( IPADDR( *(osock->src_addr)) ); } bsock->set_duped(); + bsock->set_master(osock); return bsock; } diff --git a/bacula/src/lib/bnet_server.c b/bacula/src/lib/bnet_server.c index 16993401b7..0144047d28 100644 --- a/bacula/src/lib/bnet_server.c +++ b/bacula/src/lib/bnet_server.c @@ -177,7 +177,7 @@ void bnet_thread_server(dlist *addrs, int max_clients, /* Got a connection, now accept it. */ do { clilen = sizeof(clientaddr); - newsockfd = accept(fd_ptr->fd, (struct sockaddr *)&clientaddr, &clilen); + newsockfd = baccept(fd_ptr->fd, (struct sockaddr *)&clientaddr, &clilen); newsockfd = set_socket_errno(newsockfd); } while (newsockfd == SOCKET_ERROR && (errno == EINTR || errno == EAGAIN)); if (newsockfd == SOCKET_ERROR) { diff --git a/bacula/src/lib/bpipe.c b/bacula/src/lib/bpipe.c index 92b5b85624..4c90c874a6 100644 --- a/bacula/src/lib/bpipe.c +++ b/bacula/src/lib/bpipe.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -26,6 +26,7 @@ #include "bacula.h" #include "jcr.h" +#include int execvp_errors[] = { EACCES, @@ -72,6 +73,8 @@ BPIPE *open_bpipe(char *prog, int wait, const char *mode, char *envp[]) int mode_read, mode_write, mode_shell; BPIPE *bpipe; int save_errno; + struct rlimit rl; + int64_t rlimitResult=0; if (!prog || !*prog) { /* execve(3) A component of the file does not name an existing file or file is an empty string. */ @@ -94,6 +97,16 @@ BPIPE *open_bpipe(char *prog, int wait, const char *mode, char *envp[]) } else { build_argc_argv(tprog, &bargc, bargv, MAX_ARGV); } + + /* Unable to parse the command, avoid segfault after the fork() */ + if (bargc == 0 || bargv[0] == NULL) { + free_pool_memory(tprog); + free(bpipe); + /* execve(3) A component of the file does not name an existing file or file is an empty string. */ + errno = ENOENT; + return NULL; + } + #ifdef xxxxxx printf("argc=%d\n", bargc); for (i=0; iworker_pid = fork()) { case -1: /* error */ @@ -147,11 +172,16 @@ BPIPE *open_bpipe(char *prog, int wait, const char *mode, char *envp[]) dup2(readp[1], 1); /* dup our read to his stdout */ dup2(readp[1], 2); /* and his stderr */ } -/* Note, the close log cause problems, see bug #1536 */ -/* closelog(); close syslog if open */ - for (i=3; i<=32; i++) { /* close any open file descriptors */ + +#if HAVE_FCNTL_F_CLOSEM + fcntl(3, F_CLOSEM); +#elif HAVE_CLOSEFROM + closefrom(3); +#else + for (i=rlimitResult; i >= 3; i--) { close(i); } +#endif /* Setup the environment if requested, we do not use execvpe() * because it's not wildly available @@ -163,10 +193,11 @@ BPIPE *open_bpipe(char *prog, int wait, const char *mode, char *envp[]) /* Convert errno into an exit code for later analysis */ for (i=0; i< num_execvp_errors; i++) { if (execvp_errors[i] == errno) { - exit(200 + i); /* exit code => errno */ + _exit(200 + i); /* exit code => errno */ } } - exit(255); /* unknown errno */ + /* Do not flush stdio */ + _exit(255); /* unknown errno */ default: /* parent */ break; @@ -237,18 +268,18 @@ int close_bpipe(BPIPE *bpipe) /* wait for worker child to exit */ for ( ;; ) { - Dmsg2(800, "Wait for %d opt=%d\n", bpipe->worker_pid, wait_option); + Dmsg2(100, "Wait for %d opt=%d\n", bpipe->worker_pid, wait_option); do { wpid = waitpid(bpipe->worker_pid, &chldstatus, wait_option); } while (wpid == -1 && (errno == EINTR || errno == EAGAIN)); if (wpid == bpipe->worker_pid || wpid == -1) { berrno be; stat = errno; - Dmsg3(800, "Got break wpid=%d status=%d ERR=%s\n", wpid, chldstatus, + Dmsg3(100, "Got break wpid=%d status=%d ERR=%s\n", wpid, chldstatus, wpid==-1?be.bstrerror():"none"); break; } - Dmsg3(800, "Got wpid=%d status=%d ERR=%s\n", wpid, chldstatus, + Dmsg3(100, "Got wpid=%d status=%d ERR=%s\n", wpid, chldstatus, wpid==-1?strerror(errno):"none"); if (remaining_wait > 0) { bmicrosleep(1, 0); /* wait one second */ @@ -263,17 +294,17 @@ int close_bpipe(BPIPE *bpipe) if (WIFEXITED(chldstatus)) { /* process exit()ed */ stat = WEXITSTATUS(chldstatus); if (stat != 0) { - Dmsg1(800, "Non-zero status %d returned from child.\n", stat); + Dmsg1(100, "Non-zero status %d returned from child.\n", stat); stat |= b_errno_exit; /* exit status returned */ } - Dmsg1(800, "child status=%d\n", stat & ~b_errno_exit); + Dmsg1(100, "child status=%d\n", stat & ~b_errno_exit); } else if (WIFSIGNALED(chldstatus)) { /* process died */ #ifndef HAVE_WIN32 stat = WTERMSIG(chldstatus); #else stat = 1; /* fake child status */ #endif - Dmsg1(800, "Child died from signal %d\n", stat); + Dmsg1(100, "Child died from signal %d\n", stat); stat |= b_errno_signal; /* exit signal returned */ } } @@ -281,7 +312,7 @@ int close_bpipe(BPIPE *bpipe) stop_child_timer(bpipe->timer_id); } free(bpipe); - Dmsg2(800, "returning stat=%d,%d\n", stat & ~(b_errno_exit|b_errno_signal), stat); + Dmsg2(100, "returning stat=%d,%d\n", stat & ~(b_errno_exit|b_errno_signal), stat); return stat; } @@ -368,11 +399,11 @@ int run_program(char *prog, int wait, POOLMEM *&results) } if (stat1 < 0) { berrno be; - Dmsg2(150, "Run program fgets stat=%d ERR=%s\n", stat1, be.bstrerror(errno)); + Dmsg2(100, "Run program fgets stat=%d ERR=%s\n", stat1, be.bstrerror(errno)); } else if (stat1 != 0) { - Dmsg1(150, "Run program fgets stat=%d\n", stat1); + Dmsg1(100, "Run program fgets stat=%d\n", stat1); if (bpipe->timer_id) { - Dmsg1(150, "Run program fgets killed=%d\n", bpipe->timer_id->killed); + Dmsg1(100, "Run program fgets killed=%d\n", bpipe->timer_id->killed); /* NB: I'm not sure it is really useful for run_program. Without the * following lines run_program would not detect if the program was killed * by the watchdog. */ @@ -384,7 +415,7 @@ int run_program(char *prog, int wait, POOLMEM *&results) } stat2 = close_bpipe(bpipe); stat1 = stat2 != 0 ? stat2 : stat1; - Dmsg1(150, "Run program returning %d\n", stat1); + Dmsg1(100, "Run program returning %d\n", stat1); return stat1; } @@ -436,19 +467,19 @@ int run_program_full_output(char *prog, int wait, POOLMEM *&results, char *env[] pm_strcat(tmp, buf); if (feof(bpipe->rfd)) { stat1 = 0; - Dmsg1(900, "Run program fgets stat=%d\n", stat1); + Dmsg1(100, "Run program fgets stat=%d\n", stat1); break; } else { stat1 = ferror(bpipe->rfd); } if (stat1 < 0) { berrno be; - Dmsg2(200, "Run program fgets stat=%d ERR=%s\n", stat1, be.bstrerror()); + Dmsg2(100, "Run program fgets stat=%d ERR=%s\n", stat1, be.bstrerror()); break; } else if (stat1 != 0) { - Dmsg1(900, "Run program fgets stat=%d\n", stat1); + Dmsg1(200, "Run program fgets stat=%d\n", stat1); if (bpipe->timer_id && bpipe->timer_id->killed) { - Dmsg1(250, "Run program saw fgets killed=%d\n", bpipe->timer_id->killed); + Dmsg1(100, "Run program saw fgets killed=%d\n", bpipe->timer_id->killed); break; } } @@ -460,16 +491,16 @@ int run_program_full_output(char *prog, int wait, POOLMEM *&results, char *env[] * just as the timer kills it. */ if (bpipe->timer_id && bpipe->timer_id->killed) { - Dmsg1(150, "Run program fgets killed=%d\n", bpipe->timer_id->killed); + Dmsg1(100, "Run program fgets killed=%d\n", bpipe->timer_id->killed); pm_strcpy(tmp, _("Program killed by Bacula (timeout)\n")); stat1 = ETIME; } pm_strcpy(results, tmp); - Dmsg3(1900, "resadr=0x%x reslen=%d res=%s\n", results, strlen(results), results); + Dmsg3(200, "resadr=0x%x reslen=%d res=%s\n", results, strlen(results), results); stat2 = close_bpipe(bpipe); stat1 = stat2 != 0 ? stat2 : stat1; - Dmsg1(900, "Run program returning %d\n", stat1); + Dmsg1(100, "Run program returning %d\n", stat1); bail_out: free_pool_memory(tmp); free(buf); diff --git a/bacula/src/lib/bsock.c b/bacula/src/lib/bsock.c index e0bf32b35c..1e15b03afb 100644 --- a/bacula/src/lib/bsock.c +++ b/bacula/src/lib/bsock.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,11 +20,11 @@ * Network Utility Routines * * Written by Kern Sibbald - * */ #include "bacula.h" #include "jcr.h" +#include "lz4.h" #include #include @@ -40,6 +40,37 @@ #define socketWrite(fd, buf, len) ::write(fd, buf, len) #define socketClose(fd) ::close(fd) +/* + * make a nice dump of a message + */ +void dump_bsock_msg(int sock, uint32_t msgno, const char *what, uint32_t rc, int32_t pktsize, uint32_t flags, POOLMEM *msg, int32_t msglen) +{ + char buf[54]; + bool is_ascii; + int dbglvl = DT_ASX; + + if (msglen<0) { + Dmsg4(dbglvl, "%s %d:%d SIGNAL=%s\n", what, sock, msgno, bnet_sig_to_ascii(msglen)); + // data + smartdump(msg, msglen, buf, sizeof(buf)-9, &is_ascii); + if (is_ascii) { + Dmsg5(dbglvl, "%s %d:%d len=%d \"%s\"\n", what, sock, msgno, msglen, buf); + } else { + Dmsg5(dbglvl, "%s %d:%d len=%d %s\n", what, sock, msgno, msglen, buf); + } + } +} + + +BSOCKCallback::BSOCKCallback() +{ +} + +BSOCKCallback::~BSOCKCallback() +{ +} + + /* * This is a non-class BSOCK "constructor" because we want to * call the Bacula smartalloc routines instead of new. @@ -54,11 +85,14 @@ BSOCK *new_bsock() void BSOCK::init() { memset(this, 0, sizeof(BSOCK)); + m_master = this; set_closed(); set_terminated(); m_blocking = 1; pout_msg_no = &out_msg_no; + uninstall_send_hook_cb(); msg = get_pool_memory(PM_BSOCK); + cmsg = get_pool_memory(PM_BSOCK); errmsg = get_pool_memory(PM_MESSAGE); timeout = BSOCK_TIMEOUT; } @@ -202,7 +236,7 @@ bool BSOCK::open(JCR *jcr, const char *name, char *host, char *service, ipaddr->build_address_str(curbuf, sizeof(curbuf)), build_addresses_str(addr_list, allbuf, sizeof(allbuf))); /* Open a TCP socket */ - if ((sockfd = socket(ipaddr->get_family(), SOCK_STREAM, 0)) < 0) { + if ((sockfd = socket(ipaddr->get_family(), SOCK_STREAM|SOCK_CLOEXEC, 0)) < 0) { berrno be; save_errno = errno; switch (errno) { @@ -340,6 +374,12 @@ bool BSOCK::set_locking() be.bstrerror(stat)); return false; } + if ((stat = pthread_mutex_init(&m_mmutex, NULL)) != 0) { + berrno be; + Qmsg(m_jcr, M_FATAL, 0, _("Could not init bsock attribute mutex. ERR=%s\n"), + be.bstrerror(stat)); + return false; + } m_use_locking = true; return true; } @@ -352,12 +392,122 @@ void BSOCK::clear_locking() m_use_locking = false; pthread_mutex_destroy(pm_rmutex); pthread_mutex_destroy(pm_wmutex); + pthread_mutex_destroy(&m_mmutex); pm_rmutex = NULL; pm_wmutex = NULL; return; } /* + * Do comm line compression (LZ4) of a bsock message. + * Returns: true if the compression was done + * false if no compression was done + * The "offset" defines where to start compressing the message. This + * allows passing "header" information uncompressed and the actual + * data part compressed. + * + * Note, we don't compress lines less than 20 characters because we + * want to save at least 10 characters otherwise compression doesn't + * help enough to warrant doing the decompression. + */ +bool BSOCK::comm_compress() +{ + bool compress = false; + bool compressed = false; + int offset = m_flags & 0xFF; + + /* + * Enable compress if allowed and not spooling and the + * message is long enough (>20) to get some reasonable savings. + */ + if (msglen > 20) { + compress = can_compress() && !is_spooling(); + } + m_CommBytes += msglen; /* uncompressed bytes */ + Dmsg4(DT_NETWORK|200, "can_compress=%d compress=%d CommBytes=%lld CommCompresedBytes=%lld\n", + can_compress(), compress, m_CommBytes, m_CommCompressedBytes); + if (compress) { + int clen; + int need_size; + + ASSERT2(offset <= msglen, "Comm offset bigger than message\n"); + ASSERT2(offset < 255, "Offset greater than 254\n"); + need_size = LZ4_compressBound(msglen); + if (need_size >= ((int32_t)sizeof_pool_memory(cmsg))) { + cmsg = realloc_pool_memory(cmsg, need_size + 100); + } + msglen -= offset; + msg += offset; + cmsg += offset; + clen = LZ4_compress_limitedOutput(msg, cmsg, msglen, msglen); + //Dmsg2(000, "clen=%d msglen=%d\n", clen, msglen); + /* Compression should save at least 10 characters */ + if (clen > 0 && clen + 10 <= msglen) { + +#ifdef xxx_debug + /* Debug code -- decompress and compare */ + int blen, rlen, olen; + olen = msglen; + POOLMEM *rmsg = get_pool_memory(PM_BSOCK); + blen = sizeof_pool_memory(msg) * 2; + if (blen >= sizeof_pool_memory(rmsg)) { + rmsg = realloc_pool_memory(rmsg, blen); + } + rlen = LZ4_decompress_safe(cmsg, rmsg, clen, blen); + //Dmsg4(000, "blen=%d clen=%d olen=%d rlen=%d\n", blen, clen, olen, rlen); + ASSERT(olen == rlen); + ASSERT(memcmp(msg, rmsg, olen) == 0); + free_pool_memory(rmsg); + /* end Debug code */ +#endif + + msg = cmsg; + msglen = clen; + compressed = true; + } + msglen += offset; + msg -= offset; + cmsg -= offset; + } + m_CommCompressedBytes += msglen; + return compressed; +} + + +/* + * Send a message over the network. Everything is sent in one + * write request, but depending on the mode you are using + * there will be either two or three read requests done. + * Read 1: 32 bits, gets essentially the packet length, but may + * have the upper bits set to indicate compression or + * an extended header packet. + * Read 2: 32 bits, this read is done only of BNET_HDR_EXTEND is set. + * In this case the top 16 bits of this 32 bit word are reserved + * for flags and the lower 16 bits for data. This word will be + * stored in the field "flags" in the BSOCK packet. + * Read 2 or 3: depending on if Read 2 is done. This is the data. + * + * For normal comm line compression, the whole data packet is compressed + * but not the msglen (first read). + * To do data compression rather than full comm line compression, prior to + * call send(flags) where the lower 32 bits is the offset to the data to + * be compressed. The top 32 bits are reserved for flags that can be + * set. The are: + * BNET_IS_CMD We are sending a command + * BNET_OFFSET An offset is specified (this implies data compression) + * BNET_NOCOMPRESS Inhibit normal comm line compression + * BNET_DATACOMPRESSED The data using the specified offset was + * compressed, and normal comm line compression will + * not be done. + * If any of the above bits are set, then BNET_HDR_EXTEND will be set + * in the top bits of msglen, and the full set of flags + the offset + * will be passed as a 32 bit word just after the msglen, and then + * followed by any data that is either compressed or not. + * + * Note, neither comm line nor data compression is not + * guaranteed since it may result in more data, in which case, the + * record is sent uncompressed and there will be no offset. + * On the receive side, if BNET_OFFSET is set, then the data is compressed. * * Returns: false on failure * true on success @@ -367,10 +517,12 @@ bool BSOCK::send(int aflags) int32_t rc; int32_t pktsiz; int32_t *hdrptr; + int offset; int hdrsiz; bool ok = true; int32_t save_msglen; POOLMEM *save_msg; + bool compressed; bool locked = false; if (is_closed()) { @@ -403,6 +555,15 @@ bool BSOCK::send(int aflags) return false; } + if (send_hook_cb) { + if (!send_hook_cb->bsock_send_cb()) { + Dmsg3(1, "Flowcontrol failure on %s:%s:%d\n", m_who, m_host, m_port); + Qmsg3(m_jcr, M_ERROR, 0, + _("Flowcontrol failure on %s:%s:%d\n"), + m_who, m_host, m_port); + return false; + } + } if (m_use_locking) { pP(pm_wmutex); locked = true; @@ -411,6 +572,24 @@ bool BSOCK::send(int aflags) save_msg = msg; m_flags = aflags; + offset = aflags & 0xFF; /* offset is 16 bits */ + if (offset) { + m_flags |= BNET_OFFSET; + } + if (m_flags & BNET_DATACOMPRESSED) { /* Check if already compressed */ + compressed = true; + } else if (m_flags & BNET_NOCOMPRESS) { + compressed = false; + } else { + compressed = comm_compress(); /* do requested compression */ + } + if (offset && compressed) { + m_flags |= BNET_DATACOMPRESSED; + } + if (!compressed) { + m_flags &= ~BNET_COMPRESSED; + } + /* Compute total packet length */ if (msglen <= 0) { hdrsiz = sizeof(pktsiz); @@ -423,6 +602,18 @@ bool BSOCK::send(int aflags) pktsiz = msglen + hdrsiz; } + /* Set special bits */ + if (m_flags & BNET_OFFSET) { /* if data compression on */ + compressed = false; /* no comm compression */ + } + if (compressed) { + msglen |= BNET_COMPRESSED; /* comm line compression */ + } + + if (m_flags) { + msglen |= BNET_HDR_EXTEND; /* extended header */ + } + /* * Store packet length at head of message -- note, we * have reserved an int32_t just before msg, so we can @@ -441,7 +632,7 @@ bool BSOCK::send(int aflags) clear_timed_out(); /* Full I/O done in one write */ rc = write_nbytes(this, (char *)hdrptr, pktsiz); -// if (chk_dbglvl(DT_NETWORK|1900)) dump_bsock_msg(m_fd, *pout_msg_no, "SEND", rc, msglen, m_flags, save_msg, save_msglen); + if (chk_dbglvl(DT_NETWORK|1900)) dump_bsock_msg(m_fd, *pout_msg_no, "SEND", rc, msglen, m_flags, save_msg, save_msglen); timer_start = 0; /* clear timer */ if (rc != pktsiz) { errors++; @@ -464,6 +655,8 @@ bool BSOCK::send(int aflags) } ok = false; } +// Dmsg4(000, "cmpr=%d ext=%d cmd=%d m_flags=0x%x\n", msglen&BNET_COMPRESSED?1:0, +// msglen&BNET_HDR_EXTEND?1:0, msglen&BNET_CMD_BIT?1:0, m_flags); msglen = save_msglen; msg = save_msg; if (locked) pV(pm_wmutex); @@ -480,6 +673,9 @@ bool BSOCK::fsend(const char *fmt, ...) va_list arg_ptr; int maxlen; + if (is_null(this)) { + return false; /* do not seg fault */ + } if (errors || is_terminated() || is_closed()) { return false; } @@ -522,9 +718,12 @@ int32_t BSOCK::recv() { int32_t nbytes; int32_t pktsiz; + int32_t o_pktsiz = 0; + bool compressed = false; + bool command = false; bool locked = false; - msg[0] = 0; + cmsg[0] = msg[0] = 0; msglen = 0; m_flags = 0; if (errors || is_terminated() || is_closed()) { @@ -562,6 +761,47 @@ int32_t BSOCK::recv() } pktsiz = ntohl(pktsiz); /* decode no. of bytes that follow */ + o_pktsiz = pktsiz; + /* If extension, read it */ + if (pktsiz > 0 && (pktsiz & BNET_HDR_EXTEND)) { + timer_start = watchdog_time; /* set start wait time */ + clear_timed_out(); + if ((nbytes = read_nbytes(this, (char *)&m_flags, sizeof(int32_t))) <= 0) { + timer_start = 0; /* clear timer */ + /* probably pipe broken because client died */ + if (errno == 0) { + b_errno = ENODATA; + } else { + b_errno = errno; + } + errors++; + nbytes = BNET_HARDEOF; /* assume hard EOF received */ + goto get_out; + } + timer_start = 0; /* clear timer */ + if (nbytes != sizeof(int32_t)) { + errors++; + b_errno = EIO; + Qmsg5(m_jcr, M_ERROR, 0, _("Read expected %d got %d from %s:%s:%d\n"), + sizeof(int32_t), nbytes, m_who, m_host, m_port); + nbytes = BNET_ERROR; + goto get_out; + } + pktsiz &= ~BNET_HDR_EXTEND; + m_flags = ntohl(m_flags); + } + + if (pktsiz > 0 && (pktsiz & BNET_COMPRESSED)) { + compressed = true; + pktsiz &= ~BNET_COMPRESSED; + } + + if (m_flags & BNET_IS_CMD) { + command = true; + } + if (m_flags & BNET_OFFSET) { + compressed = true; + } if (pktsiz == 0) { /* No data transferred */ timer_start = 0; /* clear timer */ @@ -575,7 +815,7 @@ int32_t BSOCK::recv() if (pktsiz < 0 || pktsiz > 1000000) { if (pktsiz > 0) { /* if packet too big */ Qmsg4(m_jcr, M_FATAL, 0, - _("Packet size=%d too big from \"%s:%s:%d. Terminating connection.\n"), + _("Packet size=%d too big from \"%s:%s:%d\". Maximum permitted 1000000. Terminating connection.\n"), pktsiz, m_who, m_host, m_port); pktsiz = BNET_TERMINATE; /* hang up */ } @@ -621,6 +861,59 @@ int32_t BSOCK::recv() nbytes = BNET_ERROR; goto get_out; } + /* If compressed uncompress it */ + if (compressed) { + int offset = 0; + int psize = nbytes * 4; + if (psize >= ((int32_t)sizeof_pool_memory(cmsg))) { + cmsg = realloc_pool_memory(cmsg, psize); + } + psize = sizeof_pool_memory(cmsg); + if (m_flags & BNET_OFFSET) { + offset = m_flags & 0xFF; + msg += offset; + msglen -= offset; + } + /* Grow buffer to max approx 4MB */ + for (int i=0; i < 7; i++) { + nbytes = LZ4_decompress_safe(msg, cmsg, msglen, psize); + if (nbytes >= 0) { + break; + } + if (psize < 65536) { + psize = 65536; + } else { + psize = psize * 2; + } + if (psize >= ((int32_t)sizeof_pool_memory(cmsg))) { + cmsg = realloc_pool_memory(cmsg, psize + 100); + } + } + if (m_flags & BNET_OFFSET) { + msg -= offset; + msglen += offset; + } + if (nbytes < 0) { + Jmsg1(m_jcr, M_ERROR, 0, "Decompress error!!!! ERR=%d\n", nbytes); + Pmsg3(000, "Decompress error!! pktsiz=%d cmsgsiz=%d nbytes=%d\n", pktsiz, + psize, nbytes); + b_errno = EIO; + errors++; + Qmsg5(m_jcr, M_ERROR, 0, _("Read expected %d got %d from %s:%s:%d\n"), + pktsiz, nbytes, m_who, m_host, m_port); + nbytes = BNET_ERROR; + goto get_out; + } + msglen = nbytes; + /* Make sure the buffer is big enough + one byte for EOS */ + if (msglen >= (int32_t)sizeof_pool_memory(msg)) { + msg = realloc_pool_memory(msg, msglen + 100); + } + /* If this is a data decompress, leave msg compressed */ + if (!(m_flags & BNET_OFFSET)) { + memcpy(msg, cmsg, msglen); + } + } /* always add a zero by to properly terminate any * string that was send to us. Note, we ensured above that the @@ -634,7 +927,10 @@ int32_t BSOCK::recv() Dsm_check(300); get_out: -// if ((chk_dbglvl(DT_NETWORK|1900))) dump_bsock_msg(m_fd, read_seqno, "RECV", nbytes, o_pktsiz, m_flags, msg, msglen); + if ((chk_dbglvl(DT_NETWORK|1900))) dump_bsock_msg(m_fd, read_seqno, "RECV", nbytes, o_pktsiz, m_flags, msg, msglen); + if (nbytes != BNET_ERROR && command) { + nbytes = BNET_COMMAND; + } if (locked) pV(pm_rmutex); return nbytes; /* return actual length of message */ @@ -725,6 +1021,7 @@ const char *BSOCK::bstrerror() int BSOCK::get_peer(char *buf, socklen_t buflen) { +#if !defined(HAVE_WIN32) if (peer_addr.sin_family == 0) { socklen_t salen = sizeof(peer_addr); int rval = (getpeername)(m_fd, (struct sockaddr *)&peer_addr, &salen); @@ -734,6 +1031,9 @@ int BSOCK::get_peer(char *buf, socklen_t buflen) return -1; return 0; +#else + return -1; +#endif } /* @@ -886,17 +1186,10 @@ void BSOCK::restore_blocking (int flags) * 0 if timeout * -1 if error */ -int BSOCK::wait_data(int sec, int usec) +int BSOCK::wait_data(int sec, int msec) { - fd_set fdset; - struct timeval tv; - - FD_ZERO(&fdset); - FD_SET((unsigned)m_fd, &fdset); for (;;) { - tv.tv_sec = sec; - tv.tv_usec = usec; - switch (select(m_fd + 1, &fdset, NULL, NULL, &tv)) { + switch (fd_wait_data(m_fd, WAIT_READ, sec, msec)) { case 0: /* timeout */ b_errno = 0; return 0; @@ -921,16 +1214,9 @@ int BSOCK::wait_data(int sec, int usec) /* * As above, but returns on interrupt */ -int BSOCK::wait_data_intr(int sec, int usec) +int BSOCK::wait_data_intr(int sec, int msec) { - fd_set fdset; - struct timeval tv; - - FD_ZERO(&fdset); - FD_SET((unsigned)m_fd, &fdset); - tv.tv_sec = sec; - tv.tv_usec = usec; - switch (select(m_fd + 1, &fdset, NULL, NULL, &tv)) { + switch (fd_wait_data(m_fd, WAIT_READ, sec, msec)) { case 0: /* timeout */ b_errno = 0; return 0; @@ -960,14 +1246,30 @@ int BSOCK::wait_data_intr(int sec, int usec) #define SHUT_RDWR 2 #endif +/* + * The JCR is canceled, set terminate for chained BSOCKs starting from master + */ +void BSOCK::cancel() +{ + master_lock(); + for (BSOCK *next = m_master; next != NULL; next = next->m_next) { + if (!next->m_closed) { + next->m_terminated = true; + next->m_timed_out = true; + } + } + master_unlock(); +} + /* * Note, this routine closes the socket, but leaves the * bsock memory in place. + * every thread is responsible of closing and destroying its own duped or not + * duped BSOCK */ void BSOCK::close() { BSOCK *bsock = this; - BSOCK *next; if (bsock->is_closed()) { return; @@ -975,33 +1277,29 @@ void BSOCK::close() if (!m_duped) { clear_locking(); } - for (; bsock; bsock = next) { - next = bsock->m_next; /* get possible pointer to next before destoryed */ - bsock->set_closed(); - bsock->set_terminated(); - if (!bsock->m_duped) { - /* Shutdown tls cleanly. */ - if (bsock->tls) { - tls_bsock_shutdown(bsock); - free_tls_connection(bsock->tls); - bsock->tls = NULL; - } + bsock->set_closed(); + bsock->set_terminated(); + if (!bsock->m_duped) { + /* Shutdown tls cleanly. */ + if (bsock->tls) { + tls_bsock_shutdown(bsock); + free_tls_connection(bsock->tls); + bsock->tls = NULL; + } - if (bsock->is_timed_out()) { - shutdown(bsock->m_fd, SHUT_RDWR); /* discard any pending I/O */ - } - /* On Windows this discards data if we did not do a close_wait() */ - socketClose(bsock->m_fd); /* normal close */ + if (bsock->is_timed_out()) { + shutdown(bsock->m_fd, SHUT_RDWR); /* discard any pending I/O */ } + /* On Windows this discards data if we did not do a close_wait() */ + socketClose(bsock->m_fd); /* normal close */ } return; } /* * Destroy the socket (i.e. release all resources) - * including and duped sockets. */ -void BSOCK::destroy() +void BSOCK::_destroy() { this->close(); /* Ensure that socket is closed */ @@ -1011,6 +1309,10 @@ void BSOCK::destroy() } else { ASSERT2(1 == 0, "Two calls to destroy socket"); /* double destroy */ } + if (cmsg) { + free_pool_memory(cmsg); + cmsg = NULL; + } if (errmsg) { free_pool_memory(errmsg); errmsg = NULL; @@ -1027,12 +1329,30 @@ void BSOCK::destroy() free(src_addr); src_addr = NULL; } - if (m_next) { - m_next->destroy(); - } free(this); } +/* + * Destroy the socket (i.e. release all resources) + * including duped sockets. + * should not be called from duped BSOCK + */ +void BSOCK::destroy() +{ + ASSERTD(reinterpret_cast(m_next) != 0xaaaaaaaaaaaaaaaa, "BSOCK::destroy() already called\n") + ASSERTD(this == m_master, "BSOCK::destroy() called by a non master BSOCK\n") + ASSERTD(!m_duped, "BSOCK::destroy() called by a duped BSOCK\n") + /* I'm the master I must destroy() all the duped BSOCKs */ + master_lock(); + BSOCK *ahead; + for (BSOCK *next = m_next; next != NULL; next = ahead) { + ahead = next->m_next; + next->_destroy(); + } + master_unlock(); + _destroy(); +} + /* Commands sent to Director */ static char hello[] = "Hello %s calling\n"; @@ -1167,7 +1487,7 @@ void BSOCK::control_bwlimit(int bytes) /* What exceed should be converted in sleep time */ int64_t usec_sleep = (int64_t)(m_nb_bytes /((double)m_bwlimit / 1000000.0)); if (usec_sleep > 100) { - bmicrosleep(0, usec_sleep); /* TODO: Check that bmicrosleep slept enough or sleep again */ + bmicrosleep(usec_sleep/1000000, usec_sleep%1000000); /* TODO: Check that bmicrosleep slept enough or sleep again */ m_last_tick = get_current_btime(); m_nb_bytes = 0; } else { diff --git a/bacula/src/lib/bsock.h b/bacula/src/lib/bsock.h index 97f84bcf2e..e2de186b52 100644 --- a/bacula/src/lib/bsock.h +++ b/bacula/src/lib/bsock.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -28,7 +28,6 @@ * * Negative msglen, is special "signal" (no data follows). * See below for SIGNAL codes. - * */ #ifndef __BSOCK_H_ @@ -41,6 +40,12 @@ class BSOCK; btimer_t *start_bsock_timer(BSOCK *bs, uint32_t wait); void stop_bsock_timer(btimer_t *wid); +class BSOCKCallback { +public: + BSOCKCallback(); + virtual ~BSOCKCallback(); + virtual bool bsock_send_cb() = 0; +}; class BSOCK { /* @@ -50,6 +55,7 @@ class BSOCK { public: uint64_t read_seqno; /* read sequence number */ POOLMEM *msg; /* message pool buffer */ + POOLMEM *cmsg; /* Compress buffer */ POOLMEM *errmsg; /* edited error message */ RES *res; /* Resource to which we are connected */ FILE *m_spool_fd; /* spooling file */ @@ -70,11 +76,29 @@ public: struct sockaddr client_addr; /* client's IP address */ struct sockaddr_in peer_addr; /* peer's IP address */ + /* when "installed", send_hook_cb->bsock_send_cb() is called before + * any ::send(). + */ + BSOCKCallback *send_hook_cb; + private: + /* m_master is used by "duped" BSOCK to access some attributes of the "parent" + * thread to have an up2date status (for example when the job is canceled, + * the "parent" BSOCK is "terminated", but the duped BSOCK is unchanged) + * In the future more attributes and method could use the "m_master" + * indirection. + * master->m_rmutex could replace pm_rmutex, idem for the (w)rite" mutex + * "m_master->error" should be incremented instead of "error", but + * this require a lock. + * + * USAGE: the parent thread MUST be sure that the child thread have quit + * before to free the "parent" BSOCK. + */ BSOCK *m_next; /* next BSOCK if duped (not actually used) */ JCR *m_jcr; /* jcr or NULL for error msgs */ pthread_mutex_t m_rmutex; /* for read locking if use_locking set */ pthread_mutex_t m_wmutex; /* for write locking if use_locking set */ + mutable pthread_mutex_t m_mmutex; /* when accessing the master/next chain */ pthread_mutex_t *pm_rmutex; /* Pointer to the read mutex */ pthread_mutex_t *pm_wmutex; /* Pointer to the write mutex */ char *m_who; /* Name of daemon to which we are talking */ @@ -91,18 +115,25 @@ private: bool m_closed: 1; /* set when socket is closed */ bool m_duped: 1; /* set if duped BSOCK */ bool m_spool: 1; /* set for spooling */ - bool m_use_locking: 1; /* set to use locking */ + bool m_compress: 1; /* set to use comm line compression */ + bool m_use_locking; /* set to use locking (out of a bitfield */ + /* to avoid race conditions) */ int64_t m_bwlimit; /* set to limit bandwidth */ int64_t m_nb_bytes; /* bytes sent/recv since the last tick */ btime_t m_last_tick; /* last tick used by bwlimit */ + uint64_t m_CommBytes; /* Bytes sent */ + uint64_t m_CommCompressedBytes; /* Compressed bytes sent */ void fin_init(JCR * jcr, int sockfd, const char *who, const char *host, int port, struct sockaddr *lclient_addr); bool open(JCR *jcr, const char *name, char *host, char *service, int port, utime_t heart_beat, int *fatal); + void master_lock() const { if (m_use_locking) pP((&m_mmutex)); }; + void master_unlock() const { if (m_use_locking) pV((&m_mmutex)); }; public: + BSOCK *m_master; /* "this" or the "parent" BSOCK if duped */ /* methods -- in bsock.c */ void init(); void free_tls(); @@ -115,7 +146,9 @@ public: bool fsend(const char*, ...); bool signal(int signal); void close(); /* close connection and destroy packet */ + void _destroy(); /* called by destroy() */ void destroy(); /* destroy socket packet */ + bool comm_compress(); /* in bsock.c */ const char *bstrerror(); /* last error on socket */ int get_peer(char *buf, socklen_t buflen); bool despool(void update_attr_spool_size(ssize_t size), ssize_t tsize); @@ -124,8 +157,8 @@ public: int set_blocking(); void restore_blocking(int flags); void set_killable(bool killable); - int wait_data(int sec, int usec=0); - int wait_data_intr(int sec, int usec=0); + int wait_data(int sec, int msec=0); + int wait_data_intr(int sec, int msec=0); bool authenticate_director(const char *name, const char *password, TLS_CONTEXT *tls_ctx, char *response, int response_len); bool set_locking(); /* in bsock.c */ @@ -150,8 +183,9 @@ public: bool is_timed_out() const { return m_timed_out; }; bool is_closed() const { return m_closed; }; bool is_open() const { return !m_closed; }; - bool is_stop() const { return errors || is_terminated() || is_closed(); } + bool is_stop() const { return errors || is_terminated() || is_closed(); }; bool is_error() { errno = b_errno; return errors; }; + bool can_compress() const { return m_compress; }; void set_data_end(int32_t FileIndex) { if (m_spool && FileIndex > m_FileIndex) { m_lastFileIndex = m_FileIndex; @@ -162,11 +196,16 @@ public: }; boffset_t get_last_data_end() { return m_last_data_end; }; int32_t get_lastFileIndex() { return m_lastFileIndex; }; + uint32_t CommBytes() { return m_CommBytes; }; + uint32_t CommCompressedBytes() { return m_CommCompressedBytes; }; void set_bwlimit(int64_t maxspeed) { m_bwlimit = maxspeed; }; bool use_bwlimit() { return m_bwlimit > 0;}; void set_spooling() { m_spool = true; }; void clear_spooling() { m_spool = false; }; + void set_compress() { m_compress = true; }; + void clear_compress() { m_compress = false; }; void set_duped() { m_duped = true; }; + void set_master(BSOCK *master) { master_lock(); m_master = master; m_next = master->m_next; master->m_next = this; master_unlock(); }; void set_timed_out() { m_timed_out = true; }; void clear_timed_out() { m_timed_out = false; }; void set_terminated() { m_terminated = true; }; @@ -174,6 +213,9 @@ public: void start_timer(int sec) { m_tid = start_bsock_timer(this, sec); }; void stop_timer() { stop_bsock_timer(m_tid); }; void swap_msgs(); + void install_send_hook_cb(BSOCKCallback *obj) { send_hook_cb=obj; }; + void uninstall_send_hook_cb() { send_hook_cb=NULL; }; + void cancel(); /* call it when JCR is canceled */ }; @@ -210,9 +252,31 @@ enum { BNET_START_RTREE = -25, /* Start restore tree mode */ BNET_END_RTREE = -26, /* End restore tree mode */ BNET_SUB_PROMPT = -27, /* Indicate we are at a subprompt */ - BNET_TEXT_INPUT = -28 /* Get text input from user */ + BNET_TEXT_INPUT = -28, /* Get text input from user */ + BNET_EXT_TERMINATE = -29, /* A Terminate condition has been met and + already reported somewhere else */ + BNET_FDCALLED = -30 /* The FD should keep the connection for a new job */ }; +/* + * These bits ares set in the packet length field. Attempt to + * keep the number of bits to a minimum and instead use the new + * flag field for passing bits using the BNET_HDR_EXTEND bit. + * Note: we must not set the high bit as that indicates a signal. + */ +#define BNET_COMPRESSED (1<<30) /* set for lz4 compressed data */ +#define BNET_HDR_EXTEND (1<<29) /* extended header */ + +/* + * The following bits are kept in flags. The high 16 bits are + * for flags, and the low 16 bits are for other info such as + * compressed data offset (BNET_OFFSET) + */ +#define BNET_IS_CMD (1<<28) /* set for command data */ +#define BNET_OFFSET (1<<27) /* Data compression offset specified */ +#define BNET_NOCOMPRESS (1<<25) /* Disable compression */ +#define BNET_DATACOMPRESSED (1<<24) /* Data compression */ + #define BNET_SETBUF_READ 1 /* Arg for bnet_set_buffer_size */ #define BNET_SETBUF_WRITE 2 /* Arg for bnet_set_buffer_size */ @@ -224,9 +288,30 @@ enum { enum { BNET_SIGNAL = -1, BNET_HARDEOF = -2, - BNET_ERROR = -3 + BNET_ERROR = -3, + BNET_COMMAND = -4, +}; + +/* + * Inter-daemon commands + * When BNET_IS_CMD is on, the next int32 is a command + */ +#define BNET_CMD_SIZE sizeof(int32_t) + +enum { + BNET_CMD_NONE = 0, /* reserved */ + BNET_CMD_ACK_HASH = 1, /* backup SD->FD SD already know this hash, don't need the block */ + BNET_CMD_UNK_HASH = 2, /* restore SD->FD hash is unknown */ + BNET_CMD_GET_HASH = 3, /* backup SD->FD SD ask FD to send the corresponding block */ + /* restore FD->SD FD ask SD to send the corresponding block */ + BNET_CMD_STO_BLOCK = 4, /* backup FD->SD FD send requested block */ + BNET_CMD_REC_ACK = 5, /* restore FD->SD FD has consumed records from the buffer */ + BNET_CMD_STP_THREAD = 6, /* restore FD->SD SD must stop thread */ + BNET_CMD_STP_FLOWCTRL = 7, /* backup FD->SD SD must stop sending flowcontrol information */ }; +const char *bnet_cmd_to_name(int val); + /* * TLS enabling values. Value is important for comparison, ie: * if (tls_remote_need < BNET_TLS_REQUIRED) { ... } diff --git a/bacula/src/lib/bsys.c b/bacula/src/lib/bsys.c index 54add7bb3f..4cfbba4adf 100644 --- a/bacula/src/lib/bsys.c +++ b/bacula/src/lib/bsys.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -533,8 +533,8 @@ void b_memset(const char *file, int line, void *mem, int val, size_t num) #if !defined(HAVE_WIN32) static int del_pid_file_ok = FALSE; -static int pid_fd = -1; #endif +static int pid_fd = -1; #ifdef HAVE_FCNTL_LOCK /* a convenient function [un]lock file using fnctl() @@ -558,7 +558,7 @@ int fcntl_lock(int fd, int code) * 2: Successs, but a previous file was found */ #if !defined(HAVE_FCNTL_LOCK) || defined(HAVE_WIN32) -int create_lock_file(char *fname, const char *progname, const char *filetype, POOLMEM **errmsg) +int create_lock_file(char *fname, const char *progname, const char *filetype, POOLMEM **errmsg, int *fd) { int ret = 1; #if !defined(HAVE_WIN32) @@ -614,31 +614,31 @@ int create_lock_file(char *fname, const char *progname, const char *filetype, PO return ret; } #else /* defined(HAVE_FCNTL_LOCK) */ -int create_lock_file(char *fname, const char *progname, const char *filetype, POOLMEM **errmsg) +int create_lock_file(char *fname, const char *progname, const char *filetype, POOLMEM **errmsg, int *fd) { int len; int oldpid; char pidbuf[20]; /* Open the pidfile for writing */ - if ((pid_fd = open(fname, O_CREAT|O_RDWR, 0640)) >= 0) { - if (fcntl_lock(pid_fd, F_WRLCK) == -1) { + if ((*fd = open(fname, O_CREAT|O_RDWR, 0640)) >= 0) { + if (fcntl_lock(*fd, F_WRLCK) == -1) { berrno be; /* already locked by someone else, try to read the pid */ - if (read(pid_fd, &pidbuf, sizeof(pidbuf)) > 0 && + if (read(*fd, &pidbuf, sizeof(pidbuf)) > 0 && sscanf(pidbuf, "%d", &oldpid) == 1) { - Mmsg(errmsg, _("%s is already running. pid=%d\nCheck file %s\n"), + Mmsg(errmsg, _("%s is already running. pid=%d, check file %s\n"), progname, oldpid, fname); } else { Mmsg(errmsg, _("Cannot lock %s file. %s ERR=%s\n"), filetype, fname, be.bstrerror()); } - close(pid_fd); - pid_fd=-1; + close(*fd); + *fd=-1; return 0; } /* write the pid */ len = sprintf(pidbuf, "%d\n", (int)getpid()); - write(pid_fd, pidbuf, len); + write(*fd, pidbuf, len); /* KEEP THE FILE OPEN TO KEEP THE LOCK !!! */ return 1; } else { @@ -658,7 +658,7 @@ void create_pid_file(char *dir, const char *progname, int port) POOLMEM *fname = get_pool_memory(PM_FNAME); Mmsg(fname, "%s/%s.%d.pid", dir, progname, port); - if (create_lock_file(fname, progname, "pid", &errmsg) == 0) { + if (create_lock_file(fname, progname, "pid", &errmsg, &pid_fd) == 0) { Emsg1(M_ERROR_TERM, 0, "%s", errmsg); /* never return */ } @@ -958,6 +958,12 @@ void allow_os_suspensions() #if HAVE_BACKTRACE && HAVE_GCC +/* if some names are not resolved you can try using : addr2line, like this + * $ addr2line -e bin/bacula-sd -a 0x43cd11 + * OR + * use the the -rdynamic option in the linker, like this + * $ LDFLAGS="-rdynamic" make setup + */ #include #include void stack_trace() @@ -1036,21 +1042,22 @@ int fs_get_free_space(const char *path, int64_t *freeval, int64_t *totalval) return -1; } +/* This function is used after a fork, the memory manager is not be initialized + * properly, so we must stay simple. + */ void setup_env(char *envp[]) { if (envp) { #if defined(HAVE_SETENV) char *p; - POOLMEM *tmp = get_pool_memory(PM_FNAME); for (int i=0; envp[i] ; i++) { - pm_strcpy(tmp, envp[i]); - p = strchr(tmp, '='); /* HOME=/tmp */ + p = strchr(envp[i], '='); /* HOME=/tmp */ if (p) { *p=0; /* HOME\0tmp\0 */ - setenv(tmp, p+1, true); + setenv(envp[i], p+1, true); + *p='='; } } - free_pool_memory(tmp); #elif defined(HAVE_PUTENV) for (int i=0; envp[i] ; i++) { putenv(envp[i]); @@ -1110,3 +1117,322 @@ bail_out: close(fd_dst); return -1; } + +/* The poll() code is currently disabled */ +#ifdef HAVE_POLL + +#include +#define NB_EVENT 1 + +int fd_wait_data(int fd, fd_wait_mode mode, int sec, int msec) +{ + int ret; + struct pollfd fds[NB_EVENT]; /* The structure for one event */ + + fds[0].fd = fd; + fds[0].events = (mode == WAIT_READ) ? POLLIN : POLLOUT; + + ret = poll(fds, NB_EVENT, sec * 1000 + msec); + + /* Check if poll actually succeed */ + switch(ret) { + case 0: /* timeout; no event detected */ + return 0; + + case -1: /* report error and abort */ + return -1; + + default: + if (fds[0].revents & POLLIN || fds[0].revents & POLLOUT) { + return 1; + + } else { + return -1; /* unexpected... */ + } + } + return -1; /* unexpected... */ +} +#else + +/* The select() code with a bigger fd_set was tested on Linux, FreeBSD and SunOS */ +#if defined(HAVE_LINUX_OS) || defined(HAVE_FREEBSD_OS) || defined(HAVE_SUN_OS) || defined(HAVE_WIN32) + #define SELECT_MAX_FD 7990 +#else + #define SELECT_MAX_FD 1023 /* For others, we keep it low */ +#endif + +int fd_wait_data(int fd, fd_wait_mode mode, int sec, int msec) +{ + + /* TODO: Allocate the fd_set when fd > SELECT_MAX_FD */ + union { + fd_set fdset; + char bfd_buf[1000]; + }; + struct timeval tv; + int ret; + + if (fd > SELECT_MAX_FD) { + Pmsg1(0, "Too many open files for the current system fd=%d\n", fd); + return -1; + } + + memset(&bfd_buf, 0, sizeof(bfd_buf)); /* FD_ZERO(&fdset) */ + FD_SET((unsigned)fd, &fdset); + + tv.tv_sec = sec; + tv.tv_usec = msec * 1000; + + if (mode == WAIT_READ) { + ret = select(fd + 1, &fdset, NULL, NULL, &tv); + + } else { /* WAIT_WRITE */ + ret = select(fd + 1, NULL, &fdset, NULL, &tv); + } + + switch (ret) { + case 0: /* timeout */ + return 0; + case -1: + return -1; /* error return */ + default: + break; + } + return 1; +} +#endif + +/* Use SOCK_CLOEXEC option when calling accept(). If not available, + * do it ourself (but with a race condition...) + */ +int baccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) +{ + int fd; +#ifdef HAVE_ACCEPT4 + fd = accept4(sockfd, addr, addrlen, SOCK_CLOEXEC); +#else + fd = accept(sockfd, addr, addrlen); + +# ifdef HAVE_DECL_FD_CLOEXEC + if (fd >= 0) { + int tmp_errno = errno + if (fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC) < 0) { + berrno be; + Dmsg2(0, "Unable to set the CLOEXEC flag on fd=%d ERR=%s\n", fd, be.bstrerror()); + } + errno = tmp_errno; + } + +# endif /* HAVE_DECL_FD_CLOEXEC */ +#endif /* HAVE_ACCEPT4 */ + return fd; +} + +#undef fopen +FILE *bfopen(const char *path, const char *mode) +{ + char options[50]; + FILE *fp; + + bstrncpy(options, mode, sizeof(options)); + +#if defined(HAVE_STREAM_CLOEXEC) + bstrncat(options, STREAM_CLOEXEC, sizeof(options)); +#endif + + fp = fopen(path, options); + +#if !defined(HAVE_STREAM_CLOEXEC) && defined(HAVE_DECL_FD_CLOEXEC) + if (fp) { + int fd = fileno(fp); + if (fd >= 0) { + int tmp_errno = errno; + if (fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC) < 0) { + berrno be; + Dmsg2(0, "Unable to set the CLOEXEC flag on fd=%d ERR=%s\n", fd, be.bstrerror()); + } + errno = tmp_errno; + } + } +#endif + return fp; +} + + +#ifdef TEST_PROGRAM +/* The main idea of the test is pretty simple, we have a writer and a reader, and + * they wait a little bit to read or send data over the fifo. + * So, for the first packets, the writer will wait, then the reader will wait + * read/write requests should always be fast. Only the time of the fd_wait_data() + * should be long. + */ +#include "findlib/namedpipe.h" +#define PIPENAME "/tmp/wait.pipe.%d" + +#define NBPACKETS 10 +#define BUFSIZE 128*512 /* The pipe size looks to be 65K */ + +typedef struct { + int nb; + pthread_t writer; + pthread_t reader; +} job; + +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +pthread_mutex_t cond_mutex = PTHREAD_MUTEX_INITIALIZER; +int nb_ready=0; + +void *th1(void *a) +{ + NamedPipe p; + int fd, r; + btime_t s, e; + ssize_t nb; + char buf[BUFSIZE]; + job *j = (job *)a; + + namedpipe_init(&p); + bsnprintf(buf, sizeof(buf), PIPENAME, j->nb); + if (namedpipe_create(&p, buf, 0600) < 0) { + berrno be; + Dmsg2(0, "R: Unable to create the fifo %s. ERR=%s\n", buf, be.bstrerror()); + namedpipe_free(&p); + exit(2); + } + fd = namedpipe_open(&p, buf, O_RDONLY); + if (fd < 0) { + berrno be; + Dmsg2(0, "R: Unable to open the fifo %s. ERR=%s\n", buf, be.bstrerror()); + return NULL; + } + P(cond_mutex); + nb_ready++; + pthread_cond_wait(&cond, &cond_mutex); + V(cond_mutex); + for (int i = 0; i < NBPACKETS; i++) { + if (i < (NBPACKETS/2)) { + bmicrosleep(5, 0); + } + s = get_current_btime(); + r = fd_wait_data(fd, WAIT_READ, 10, 0); + if (r > 0) { + e = get_current_btime(); + Dmsg2(0, "Wait to read pkt %d %lldms\n",i, (int64_t) (e - s)); + + if (i <= NBPACKETS/2) { + ASSERT2((e-s) < 10000, "In the 1st phase, we are blocking the process"); + } else { + ASSERT2((e-s) > 10000, "In the 2nd phase, the writer is slowing down things"); + } + + s = get_current_btime(); + nb = read(fd, buf, sizeof(buf)); + e = get_current_btime(); + Dmsg3(0, "Read pkt %d %d bytes in %lldms\n",i, (int)nb, (int64_t) (e - s)); + ASSERT2((e-s) < 10000, "The read operation should be FAST"); + } + } + namedpipe_free(&p); + return NULL; +} + +void *th2(void *a) +{ + NamedPipe p; + btime_t s, e; + job *j = (job *)a; + char buf[BUFSIZE]; + int fd; + ssize_t nb; + + bsnprintf(buf, sizeof(buf), PIPENAME, j->nb); + namedpipe_init(&p); + if (namedpipe_create(&p, buf, 0600) < 0) { + berrno be; + Dmsg2(0, "W: Unable to create the fifo %s. ERR=%s\n", buf, be.bstrerror()); + namedpipe_free(&p); + exit(2); + } + + fd = namedpipe_open(&p, buf, O_WRONLY); + if (fd < 0) { + berrno be; + Dmsg2(0, "W: Unable to open the fifo %s. ERR=%s\n", buf, be.bstrerror()); + namedpipe_free(&p); + exit(2); + } + + P(cond_mutex); + nb_ready++; + pthread_cond_wait(&cond, &cond_mutex); + V(cond_mutex); + + unlink(buf); + + for (int i=0; i < NBPACKETS; i++) { + if (i > (NBPACKETS/2)) { + bmicrosleep(5, 0); + } + s = get_current_btime(); + if (fd_wait_data(fd, WAIT_WRITE, 10, 0) > 0) { + e = get_current_btime(); + Dmsg2(0, "Wait to write pkt %d %lldms\n",i, (int64_t) (e - s)); + + if (i == 0 || i > NBPACKETS/2) { /* The first packet doesn't count */ + ASSERT2((e-s) < 100000, "In the 2nd phase, it's fast to send, we are the blocker"); + } else { + ASSERT2((e-s) > 100000, "In the 1st phase, we wait for the reader"); + } + + s = get_current_btime(); + nb = write(fd, buf, sizeof(buf)); + e = get_current_btime(); + Dmsg3(0, "Wrote pkt %d %d bytes in %lldms\n", i, (int)nb, (int64_t) (e - s)); + ASSERT2((e-s) < 100000, "The write operation should never block"); + } + } + namedpipe_free(&p); + return NULL; +} + +int main(int argc, char **argv) +{ + job pthread_list[10000]; + int j = (argc >= 2) ? atoi(argv[1]) : 1; + int maxfd = (argc == 3) ? atoi(argv[2]) : 0; + + j = MIN(10000, j); + + lmgr_init_thread(); + set_debug_flags((char *)"h"); + + for (int i=3; i < maxfd; i++) { + open("/dev/null", O_RDONLY); + } + + for (int i=0; i < j; i++) { + pthread_list[i].nb=i; + pthread_create(&pthread_list[i].writer, NULL, th2, &pthread_list[i]); + pthread_create(&pthread_list[i].reader, NULL, th1, &pthread_list[i]); + } + + while (nb_ready < j*2) { + bmicrosleep(1, 0); + } + + Dmsg0(0, "All threads are started\n"); + P(cond_mutex); + pthread_cond_broadcast(&cond); + V(cond_mutex); + + for (int i=0; i < j; i++) { + pthread_join(pthread_list[i].writer, NULL); + pthread_join(pthread_list[i].reader, NULL); + } + + for (int i=3; i < maxfd; i++) { + close(i); + } + return 0; +} +#endif diff --git a/bacula/src/lib/btimers.c b/bacula/src/lib/btimers.c index 131501fe02..ff04d4c4ff 100644 --- a/bacula/src/lib/btimers.c +++ b/bacula/src/lib/btimers.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -74,7 +74,6 @@ btimer_t *start_child_timer(JCR *jcr, pid_t pid, uint32_t wait) void stop_child_timer(btimer_t *wid) { if (wid == NULL) { - Dmsg0(dbglvl, "stop_child_timer called with NULL btimer_id\n"); return; } Dmsg2(dbglvl, "Stop child timer %p pid %d\n", wid, wid->pid); @@ -106,7 +105,7 @@ static void callback_child_timer(watchdog_t *self) * be less than 5 seconds) */ kill(wid->pid, SIGTERM); - self->interval = 5; + self->interval = 10; } else { /* This is the second call - terminate with prejudice. */ Dmsg2(dbglvl, "watchdog %p kill PID %d\n", self, wid->pid); @@ -186,7 +185,6 @@ btimer_t *start_bsock_timer(BSOCK *bsock, uint32_t wait) void stop_bsock_timer(btimer_t *wid) { if (wid == NULL) { - Dmsg0(900, "stop_bsock_timer called with NULL btimer_id\n"); return; } Dmsg3(dbglvl, "Stop bsock timer %p tid=%p at %d.\n", wid, wid->tid, time(NULL)); @@ -200,7 +198,6 @@ void stop_bsock_timer(btimer_t *wid) void stop_thread_timer(btimer_t *wid) { if (wid == NULL) { - Dmsg0(dbglvl, "stop_thread_timer called with NULL btimer_id\n"); return; } Dmsg2(dbglvl, "Stop thread timer %p tid=%p.\n", wid, wid->tid); @@ -242,7 +239,7 @@ static btimer_t *btimer_start_common(uint32_t wait) return NULL; } wid->wd->data = wid; - wid->killed = FALSE; + wid->killed = false; return wid; } diff --git a/bacula/src/lib/bwlimit.c b/bacula/src/lib/bwlimit.c new file mode 100644 index 0000000000..167992685c --- /dev/null +++ b/bacula/src/lib/bwlimit.c @@ -0,0 +1,64 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "bacula.h" +#include "bwlimit.h" + +#define ONE_SEC 1000000L /* number of microseconds in a second */ + +void bwlimit::control_bwlimit(int bytes) +{ + btime_t now, temp; + if (bytes == 0 || m_bwlimit == 0) { + return; + } + + lock_guard lg(m_bw_mutex); /* Release the mutex automatically when we quit the function*/ + now = get_current_btime(); /* microseconds */ + temp = now - m_last_tick; /* microseconds */ + + m_nb_bytes += bytes; + if (temp < 0 || temp > (ONE_SEC*10)) { /* Take care of clock problems (>10s) or back in time */ + m_nb_bytes = bytes; + m_last_tick = now; + return; + } + + /* Less than 0.1ms since the last call, see the next time */ + if (temp < 100) { + return; + } + + /* Remove what was authorised to be written in temp us */ + m_nb_bytes -= (int64_t)(temp * ((double)m_bwlimit / ONE_SEC)); + + if (m_nb_bytes < 0) { + m_nb_bytes = 0; + } + + /* What exceed should be converted in sleep time */ + int64_t usec_sleep = (int64_t)(m_nb_bytes /((double)m_bwlimit / ONE_SEC)); + if (usec_sleep > 100) { + bmicrosleep(usec_sleep / ONE_SEC, usec_sleep % ONE_SEC); + m_last_tick = get_current_btime(); + m_nb_bytes = 0; + } else { + m_last_tick = now; + } +} diff --git a/bacula/src/lib/bwlimit.h b/bacula/src/lib/bwlimit.h new file mode 100644 index 0000000000..ec492edcaf --- /dev/null +++ b/bacula/src/lib/bwlimit.h @@ -0,0 +1,43 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef BWLIMIT_H +#define BWLIMIT_H + +class bwlimit: public SMARTALLOC +{ +private: + int64_t m_bwlimit; /* set to limit bandwidth */ + int64_t m_nb_bytes; /* bytes sent/recv since the last tick */ + btime_t m_last_tick; /* last tick used by bwlimit */ + pthread_mutex_t m_bw_mutex; + +public: + bwlimit(int64_t speed=0): m_bwlimit(speed), m_nb_bytes(0), m_last_tick(0) { + pthread_mutex_init(&m_bw_mutex, NULL); + }; + ~bwlimit() { + pthread_mutex_destroy(&m_bw_mutex); + }; + + void control_bwlimit(int bytes); + void set_bwlimit(int64_t maxspeed) { m_bwlimit = maxspeed; }; + bool use_bwlimit() { return m_bwlimit > 0;}; +}; +#endif diff --git a/bacula/src/lib/cmd_parser.h b/bacula/src/lib/cmd_parser.h new file mode 100644 index 0000000000..750a254d45 --- /dev/null +++ b/bacula/src/lib/cmd_parser.h @@ -0,0 +1,196 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef CMD_PARSER_H +#define CMD_PARSER_H + +extern int parse_args(POOLMEM *cmd, POOLMEM **args, int *argc, + char **argk, char **argv, int max_args); +extern int parse_args_only(POOLMEM *cmd, POOLMEM **args, int *argc, + char **argk, char **argv, int max_args); + +class cmd_parser { +public: + POOLMEM *args; + POOLMEM *cmd; /* plugin command line */ + POOLMEM *org; /* original command line */ + + char **argk; /* Argument keywords */ + char **argv; /* Argument values */ + int argc; /* Number of arguments */ + int max_cmd; /* Max number of arguments */ + bool handle_plugin_name; /* Search for : */ + + cmd_parser(bool handle_plugin_name=true) + : handle_plugin_name(handle_plugin_name) + { + org = get_pool_memory(PM_FNAME); + args = get_pool_memory(PM_FNAME); + cmd = get_pool_memory(PM_FNAME); + *args = *org = *cmd = 0; + argc = 0; + max_cmd = MAX_CMD_ARGS; + argk = argv = NULL; + }; + + virtual ~cmd_parser() { + free_pool_memory(org); + free_pool_memory(cmd); + free_pool_memory(args); + if (argk) { + free(argk); + } + if (argv) { + free(argv); + } + } + + /* + * Given a single keyword, find it in the argument list, but + * it must have a value + * Returns: -1 if not found or no value + * list index (base 0) on success + */ + int find_arg_with_value(const char *keyword) + { + for (int i=handle_plugin_name?1:0; i= next_fd; i--) { close(i); } -#endif +#endif /* Move to root directory. For debug we stay * in current directory so dumps go there. diff --git a/bacula/src/lib/edit.c b/bacula/src/lib/edit.c index d7f43b4dec..48d8e88be7 100644 --- a/bacula/src/lib/edit.c +++ b/bacula/src/lib/edit.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. diff --git a/bacula/src/lib/flist.c b/bacula/src/lib/flist.c new file mode 100644 index 0000000000..8d5b8e38fe --- /dev/null +++ b/bacula/src/lib/flist.c @@ -0,0 +1,152 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula fifo list routines + * + * flist is a simple malloc'ed array of pointers. Derived from alist. + * + * Kern Sibbald, August 2014 + * + */ + +#include "bacula.h" + +void *flist::dequeue() +{ + void *item; + + if (num_items == 0) { + return NULL; + } + num_items--; + item = items[get_item]; + items[get_item++] = NULL; + if (get_item >= max_items) { + get_item = 0; + } + return item; +} + + +/* + * Queue an item to the list + */ +bool flist::queue(void *item) +{ + if (num_items == max_items) { + return false; + } + num_items++; + items[add_item++] = item; + if (add_item >= max_items) { + add_item = 0; + } + return true; +} + +/* Destroy the list and its contents */ +void flist::destroy() +{ + if (num_items && own_items) { + for (int i=0; imylist.init(); + + printf("Manual allocation/destruction of list:\n"); + + for (i=0; i<20; i++) { + sprintf(buf, "This is item %d", i); + p = bstrdup(buf); + if (fileset->mylist.queue(p)) { + printf("Added item = %s\n", p); + } else { + q = (char *)fileset->mylist.dequeue(); + printf("Dequeue item = %s\n", q); + free(q); + if (fileset->mylist.queue(p)) { + printf("Added item = %s\n", p); + } else { + printf("Big problem could not queue item %d %s\n", i, p); + } + } + } + while ((q=(char *)fileset->mylist.dequeue())) { + printf("Dequeue item = %s\n", q); + free(q); + } + for (i=1; !fileset->mylist.empty(); i++) { + q = (char *)fileset->mylist.dequeue(); + if (!q) { + break; + } + printf("Item %d = %s\n", i, q); + free(q); + } + fileset->mylist.destroy(); + free(fileset); + + printf("Allocation/destruction using new delete\n"); + mlist = New(flist(10)); + + for (i=0; i<20; i++) { + sprintf(buf, "This is item %d", i); + p = bstrdup(buf); + if (!mlist->queue(p)) { + free(p); + break; + } + } + for (i=1; !mlist->empty(); i++) { + p = (char *)mlist->dequeue(); + printf("Item %d = %s\n", i, p); + free(p); + } + + delete mlist; + + sm_dump(false); /* test program */ + +} +#endif diff --git a/bacula/src/lib/flist.h b/bacula/src/lib/flist.h new file mode 100644 index 0000000000..cb0ddf69c4 --- /dev/null +++ b/bacula/src/lib/flist.h @@ -0,0 +1,95 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Kern Sibbald, August 2014. + */ + +/* Second arg of init */ +enum { + owned_by_flist = true, + not_owned_by_flist = false +}; + +/* + * Fifo list -- derived from alist + */ +class flist : public SMARTALLOC { + void **items; + int num_items; + int max_items; + int add_item; + int get_item; + bool own_items; +public: + flist(int num = 10, bool own=false); + ~flist(); + void init(int num = 10, bool own=false); + bool queue(void *item); + void *dequeue(); + bool empty() const; + bool full() const; + int size() const; + void destroy(); +}; + +inline bool flist::empty() const +{ + return num_items == 0; +} + +inline bool flist::full() const +{ + return num_items == max_items; +} + +/* + * This allows us to do explicit initialization, + * allowing us to mix C++ classes inside malloc'ed + * C structures. Define before called in constructor. + */ +inline void flist::init(int num, bool own) +{ + items = NULL; + num_items = 0; + max_items = num; + own_items = own; + add_item = 0; + get_item = 0; + items = (void **)malloc(max_items * sizeof(void *)); +} + +/* Constructor */ +inline flist::flist(int num, bool own) +{ + init(num, own); +} + +/* Destructor */ +inline flist::~flist() +{ + destroy(); +} + + + +/* Current size of list */ +inline int flist::size() const +{ + return num_items; +} diff --git a/bacula/src/lib/htable.c b/bacula/src/lib/htable.c index e5ceea1e1c..8f4624a672 100644 --- a/bacula/src/lib/htable.c +++ b/bacula/src/lib/htable.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -150,7 +150,7 @@ void htable::init(void *item, void *link, int tsize) { int pwr; - memset(this, 0, sizeof(htable)); + bmemzero(this, sizeof(htable)); if (tsize < 31) { tsize = 31; } @@ -164,7 +164,7 @@ void htable::init(void *item, void *link, int tsize) buckets = 1<max_items = big->buckets * 4; /* Create a bigger hash table */ big->table = (hlink **)malloc(big->buckets * sizeof(hlink *)); - memset(big->table, 0, big->buckets * sizeof(hlink *)); + bmemzero(big->table, big->buckets * sizeof(hlink *)); big->walkptr = NULL; big->walk_index = 0; /* Insert all the items in the new hash table */ diff --git a/bacula/src/lib/ini.c b/bacula/src/lib/ini.c index b4da3a0932..c7610c3493 100644 --- a/bacula/src/lib/ini.c +++ b/bacula/src/lib/ini.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -181,7 +181,7 @@ bool ConfigFile::dump_string(const char *buf, int32_t len) make_unique_filename(&out_fname, (int)(intptr_t)this, (char*)"configfile"); } - fp = fopen(out_fname, "wb"); + fp = bfopen(out_fname, "wb"); if (!fp) { return ret; } @@ -206,7 +206,7 @@ bool ConfigFile::serialize(const char *fname) return ret; } - fp = fopen(fname, "w"); + fp = bfopen(fname, "w"); if (!fp) { return ret; } @@ -279,8 +279,15 @@ int ConfigFile::dump_results(POOLMEM **buf) tmp2 = get_pool_memory(PM_MESSAGE); for (int i=0; items[i].name ; i++) { + bool process= items[i].found; if (items[i].found) { items[i].handler(NULL, this, &items[i]); + } + if (!items[i].found && items[i].required && items[i].default_value) { + pm_strcpy(this->edit, items[i].default_value); + process = true; + } + if (process) { if (items[i].comment && *items[i].comment) { Mmsg(tmp, "# %s\n", items[i].comment); pm_strcat(buf, tmp); @@ -731,7 +738,7 @@ struct ini_items test_items[] = { int32_t r_last; int32_t r_first; RES_HEAD **res_head; -void save_resource(int type, RES_ITEM *items, int pass){} +bool save_resource(RES_HEAD **rhead, int type, RES_ITEM *items, int pass){} void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, ...), void *sock){} void free_resource(RES *rres, int type){} union URES { diff --git a/bacula/src/lib/jcr.c b/bacula/src/lib/jcr.c index 3ec16c2bb0..3f76cf5e81 100644 --- a/bacula/src/lib/jcr.c +++ b/bacula/src/lib/jcr.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -78,7 +78,7 @@ static pthread_key_t jcr_key; /* Pointer to jcr for each thread */ pthread_once_t key_once = PTHREAD_ONCE_INIT; -static char Job_status[] = "Status Job=%s JobStatus=%d\n"; +static char Job_status[] = "Status JobId=%ld JobStatus=%d\n"; void lock_jobs() @@ -289,6 +289,9 @@ void job_end_push(JCR *jcr, void job_end_cb(JCR *jcr,void *), void *ctx) jcr->job_end_push.append(ctx); } +/* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ +void (*MA1512_reload_job_end_cb)(JCR *,void *) = NULL; + /* Pop each job_end subroutine and call it */ static void job_end_pop(JCR *jcr) { @@ -297,7 +300,20 @@ static void job_end_pop(JCR *jcr) for (int i=jcr->job_end_push.size()-1; i > 0; ) { ctx = jcr->job_end_push.get(i--); job_end_cb = (void (*)(JCR *,void *))jcr->job_end_push.get(i--); - job_end_cb(jcr, ctx); + /* check for bug MA1512, MA1632 MA1639, + * today, job_end_cb can only be reload_job_end_cb() from DIR */ + if (job_end_cb != MA1512_reload_job_end_cb && MA1512_reload_job_end_cb != NULL) { + Tmsg2(0, "Bug 'job_end_pop' detected, skip ! job_end_cb=0x%p ctx=0x%p\n", job_end_cb, ctx); + Tmsg0(0, "Display job_end_push list\n"); + for (int j=jcr->job_end_push.size()-1; j > 0; ) { + void *ctx2 = jcr->job_end_push.get(j--); + void *job_end_cb2 = jcr->job_end_push.get(j--); + Tmsg3(0, "Bug 'job_end_pop' entry[%d] job_end_cb=0x%p ctx=0x%p\n", j+1, job_end_cb2, ctx2); + } + } else + { + job_end_cb(jcr, ctx); + } } } @@ -333,7 +349,7 @@ JCR *new_jcr(int size, JCR_free_HANDLER *daemon_free_jcr) Jmsg1(NULL, M_ABORT, 0, _("pthread_once failed. ERR=%s\n"), be.bstrerror(status)); } jcr = (JCR *)malloc(size); - memset(jcr, 0, size); + bmemzero(jcr, size); /* Note for the director, this value is changed in jobq.c */ jcr->my_thread_id = pthread_self(); jcr->msg_queue = New(dlist(item, &item->link)); @@ -354,6 +370,9 @@ JCR *new_jcr(int size, JCR_free_HANDLER *daemon_free_jcr) jcr->errmsg[0] = 0; jcr->comment = get_pool_memory(PM_FNAME); jcr->comment[0] = 0; + jcr->StatusErrMsg = get_pool_memory(PM_FNAME); + jcr->StatusErrMsg[0] = 0; + jcr->job_uid = -1; /* Setup some dummy values */ bstrncpy(jcr->Job, "*System*", sizeof(jcr->Job)); jcr->JobId = 0; @@ -421,31 +440,20 @@ static void free_common_jcr(JCR *jcr) } /* do this after closing messages */ - if (jcr->client_name) { - free_pool_memory(jcr->client_name); - jcr->client_name = NULL; - } - - if (jcr->attr) { - free_pool_memory(jcr->attr); - jcr->attr = NULL; - } + free_and_null_pool_memory(jcr->JobIds); + free_and_null_pool_memory(jcr->client_name); + free_and_null_pool_memory(jcr->attr); + free_and_null_pool_memory(jcr->VolumeName); + free_and_null_pool_memory(jcr->errmsg); + free_and_null_pool_memory(jcr->StatusErrMsg); if (jcr->sd_auth_key) { free(jcr->sd_auth_key); jcr->sd_auth_key = NULL; } - if (jcr->VolumeName) { - free_pool_memory(jcr->VolumeName); - jcr->VolumeName = NULL; - } free_bsock(jcr->dir_bsock); - if (jcr->errmsg) { - free_pool_memory(jcr->errmsg); - jcr->errmsg = NULL; - } if (jcr->where) { free(jcr->where); jcr->where = NULL; @@ -625,7 +633,7 @@ void JCR::my_thread_send_signal(int sig) this->exiting = true; } else if (!this->is_killable()) { - Dmsg1(10, "Warning, can't send kill to jid=%d\n", this->JobId); + Dmsg1(10, "Warning, cannot send kill to jid=%d marked not killable.\n", this->JobId); } get_out: this->unlock(); @@ -867,7 +875,7 @@ static int get_status_priority(int JobStatus) bool JCR::sendJobStatus() { if (dir_bsock) { - return dir_bsock->fsend(Job_status, Job, JobStatus); + return dir_bsock->fsend(Job_status, JobId, JobStatus); } return true; } @@ -880,7 +888,7 @@ bool JCR::sendJobStatus(int aJobStatus) if (!is_JobStatus(aJobStatus)) { setJobStatus(aJobStatus); if (dir_bsock) { - return dir_bsock->fsend(Job_status, Job, JobStatus); + return dir_bsock->fsend(Job_status, JobId, JobStatus); } } return true; @@ -892,14 +900,18 @@ void JCR::setJobStarted() job_started_time = time(NULL); } +static pthread_mutex_t status_lock = PTHREAD_MUTEX_INITIALIZER; + void JCR::setJobStatus(int newJobStatus) { int priority, old_priority; int oldJobStatus = JobStatus; + + P(status_lock); priority = get_status_priority(newJobStatus); old_priority = get_status_priority(oldJobStatus); - Dmsg2(800, "set_jcr_job_status(%s, %c)\n", Job, newJobStatus); + Dmsg2(800, "set_jcr_job_status(%ld, %c)\n", JobId, newJobStatus); /* Update wait_time depending on newJobStatus and oldJobStatus */ update_wait_time(this, newJobStatus); @@ -908,7 +920,7 @@ void JCR::setJobStatus(int newJobStatus) * For a set of errors, ... keep the current status * so it isn't lost. For all others, set it. */ - Dmsg2(800, "OnEntry JobStatus=%c newJobstatus=%c\n", oldJobStatus, newJobStatus); + Dmsg2(800, "OnEntry JobStatus=%c newJobstatus=%c\n", (oldJobStatus==0)?'0':oldJobStatus, newJobStatus); /* * If status priority is > than proposed new status, change it. * If status priority == new priority and both are zero, take @@ -919,14 +931,15 @@ void JCR::setJobStatus(int newJobStatus) if (priority > old_priority || ( priority == 0 && old_priority == 0)) { Dmsg4(800, "Set new stat. old: %c,%d new: %c,%d\n", - JobStatus, old_priority, newJobStatus, priority); + (oldJobStatus==0)?'0':oldJobStatus, old_priority, newJobStatus, priority); JobStatus = newJobStatus; /* replace with new status */ } if (oldJobStatus != JobStatus) { - Dmsg2(800, "leave setJobStatus old=%c new=%c\n", oldJobStatus, newJobStatus); + Dmsg2(800, "leave setJobStatus old=%c new=%c\n", (oldJobStatus==0)?'0':oldJobStatus, newJobStatus); // generate_plugin_event(this, bEventStatusChange, NULL); } + V(status_lock); } #ifdef TRACE_JCR_CHAIN @@ -1175,7 +1188,7 @@ extern "C" void timeout_handler(int sig) */ #define MAX_DBG_HOOK 10 static dbg_jcr_hook_t *dbg_jcr_hooks[MAX_DBG_HOOK]; -static int dbg_jcr_handler_count; +static int dbg_jcr_handler_count=0; void dbg_jcr_add_hook(dbg_jcr_hook_t *hook) { diff --git a/bacula/src/lib/lex.c b/bacula/src/lib/lex.c index 8221698d0d..5b4780aab7 100644 --- a/bacula/src/lib/lex.c +++ b/bacula/src/lib/lex.c @@ -29,6 +29,19 @@ /* Debug level for this source file */ static const int dbglvl = 5000; +/* + * Return false if the end of the line contains anything other + * than spaces, or a semicolon or a comment. + */ +bool lex_check_eol(LEX *lf) +{ + char *ch = lf->line+lf->col_no; + while (*ch != '\0' && *ch != '#' && B_ISSPACE(*ch) && *ch != ';') { + ch++; + } + return *ch == '\0' || *ch == '#' || *ch == ';'; +} + /* * Scan to "logical" end of line. I.e. end of line, * or semicolon, but stop on T_EOB (same as end of @@ -109,6 +122,12 @@ int lex_set_error_handler_error_type(LEX *lf, int err_type) return old; } +/* Store passwords in clear text or with MD5 encoding */ +void lex_store_clear_passwords(LEX *lf) +{ + lf->options |= LOPT_NO_MD5; +} + /* * Free the current file, and retrieve the contents * of the previous packet if any. @@ -442,7 +461,7 @@ static uint64_t scan_pint64(LEX *lf, char *str) int lex_get_token(LEX *lf, int expect) { - int ch; + int ch, nch; int token = T_NONE; bool esc_next = false; /* Unicode files, especially on Win32, may begin with a "Byte Order Mark" @@ -485,6 +504,12 @@ lex_get_token(LEX *lf, int expect) token = T_EOF; Dmsg0(dbglvl, "got L_EOF set token=T_EOF\n"); break; + case '\\': + nch = lex_get_char(lf); + if (nch == ' ' || nch == '\n' || nch == '\r' || nch == L_EOL) { + lf->ch = L_EOL; /* force end of line */ + } + break; case '#': lf->state = lex_comment; break; diff --git a/bacula/src/lib/lex.h b/bacula/src/lib/lex.h index a6aeb91348..ae73ad1371 100644 --- a/bacula/src/lib/lex.h +++ b/bacula/src/lib/lex.h @@ -86,6 +86,7 @@ enum lex_state { #define LOPT_NO_IDENT 0x1 /* No Identifiers -- use string */ #define LOPT_STRING 0x2 /* Force scan for string */ #define LOPT_NO_EXTERN 0x4 /* Don't follow @ command */ +#define LOPT_NO_MD5 0x8 /* Do not encode passwords with MD5 */ class BPIPE; /* forward reference */ @@ -129,5 +130,6 @@ typedef void (LEX_ERROR_HANDLER)(const char *file, int line, LEX *lc, const char void scan_to_eol(LEX *lc); int scan_to_next_not_eol(LEX * lc); +void lex_store_clear_passwords(LEX *lf); #endif /* _LEX_H */ diff --git a/bacula/src/lib/lib.h b/bacula/src/lib/lib.h index 9ad722b079..88a55cac3b 100644 --- a/bacula/src/lib/lib.h +++ b/bacula/src/lib/lib.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -29,7 +29,9 @@ #include "lockmgr.h" #include "alist.h" #include "dlist.h" +#include "flist.h" #include "rblist.h" +#include "worker.h" #include "base64.h" #include "bits.h" #include "btime.h" @@ -42,6 +44,7 @@ #include "openssl.h" #include "lex.h" #include "parse_conf.h" +#include "bjson.h" #include "tls.h" #include "address_conf.h" #include "bsock.h" @@ -61,5 +64,6 @@ #include "guid_to_name.h" #include "htable.h" #include "sellist.h" +#include "output.h" #include "protos.h" #include "bget_msg.h" diff --git a/bacula/src/lib/lockmgr.c b/bacula/src/lib/lockmgr.c index 81c54855ab..7ecd201155 100644 --- a/bacula/src/lib/lockmgr.c +++ b/bacula/src/lib/lockmgr.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -16,7 +16,6 @@ Bacula(R) is a registered trademark of Kern Sibbald. */ - /* How to use mutex with bad order usage detection ------------------------------------------------ @@ -598,6 +597,14 @@ void lmgr_unregister_thread(lmgr_thread_t *item) lmgr_p(&lmgr_global_mutex); { global_mgr->remove(item); +#ifdef DEVELOPER + for(int i=0; i<=item->current; i++) { + lmgr_lock_t *lock = &item->lock_list[i]; + if (lock->state == LMGR_LOCK_GRANTED) { + ASSERT2(0, "Thread exits with granted locks"); + } + } +#endif } lmgr_v(&lmgr_global_mutex); } @@ -762,6 +769,15 @@ inline lmgr_thread_t *lmgr_get_thread_info() } } +/* + * Know if the current thread is registred (used when we + * do not control thread creation) + */ +bool lmgr_thread_is_initialized() +{ + return pthread_getspecific(lmgr_key) != NULL; +} + /* On windows, the thread id is a struct, and sometime (for debug or openssl), * we need a int */ @@ -899,18 +915,24 @@ int bthread_kill(pthread_t thread, int sig, const char *file, int line) { bool thread_found_in_process=false; - - /* We doesn't allow to send signal to ourself */ - ASSERT(!pthread_equal(thread, pthread_self())); + int ret=-1; + /* We dont allow to send signal to ourself */ + if (pthread_equal(thread, pthread_self())) { + ASSERTD(!pthread_equal(thread, pthread_self()), "Wanted to pthread_kill ourself"); + Dmsg3(10, "%s:%d send kill to self thread %p\n", file, line, thread); + errno = EINVAL; + return -1; + } /* This loop isn't very efficient with dozens of threads but we don't use - * signal very much, and this feature is for testing only + * signal very much */ lmgr_p(&lmgr_global_mutex); { lmgr_thread_t *item; foreach_dlist(item, global_mgr) { if (pthread_equal(thread, item->thread_id)) { + ret = pthread_kill(thread, sig); thread_found_in_process=true; break; } @@ -918,13 +940,13 @@ int bthread_kill(pthread_t thread, int sig, } lmgr_v(&lmgr_global_mutex); - /* Sending a signal to non existing thread can create problem - * so, we can stop here. - */ - ASSERT2(thread_found_in_process, "Wanted to pthread_kill non-existant thread"); - - Dmsg3(100, "%s:%d send kill to existing thread %p\n", file, line, thread); - return pthread_kill(thread, sig); + /* Sending a signal to non existing thread can create problem */ + if (!thread_found_in_process) { + ASSERTD(thread_found_in_process, "Wanted to pthread_kill non-existant thread"); + Dmsg3(10, "%s:%d send kill to non-existant thread %p\n", file, line, thread); + errno=ECHILD; + } + return ret; } /* @@ -1159,6 +1181,41 @@ void dbg_print_lock(FILE *fp) #endif /* USE_LOCKMGR */ +#ifdef HAVE_LINUX_OS +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#endif + +/* + * Set the Thread Id of the current thread to limit I/O operations + */ +int bthread_change_uid(uid_t uid, gid_t gid) +{ +#if defined(HAVE_WIN32) || defined(HAVE_WIN64) + /* TODO: Check the cygwin code for the implementation of setuid() */ + errno = ENOSYS; + return -1; + +#elif defined(HAVE_LINUX_OS) + /* It can be also implemented with setfsuid() and setfsgid() */ + int ret=0; + ret = syscall(SYS_setregid, getgid(), gid); + if (ret == -1) { + return -1; + } + return syscall(SYS_setreuid, getuid(), uid); + +#elif defined(HAVE_FREEBSD_OS) || defined(HAVE_DARWIN_OS) + return pthread_setugid_np(uid, gid); + +#endif + errno = ENOSYS; + return -1; +} + + #ifdef _TEST_IT #include "lockmgr.h" @@ -1226,6 +1283,35 @@ void *mix_rwl_mutex(void *temp) } +void *thuid(void *temp) +{ + char buf[512]; +// if (restrict_job_permissions("eric", "users", buf, sizeof(buf)) < 0) { + if (bthread_change_uid(2, 100) == -1) { + berrno be; + fprintf(stderr, "Unable to change the uid err=%s\n", be.bstrerror()); + } else { + fprintf(stderr, "UID set! %d:%d\n", (int)getuid(), (int)getgid()); + mkdir("/tmp/adirectory", 0755); + system("touch /tmp/afile"); + system("id"); + fclose(fopen("/tmp/aaa", "a")); + } + if (bthread_change_uid(0, 0) == -1) { + berrno be; + fprintf(stderr, "Unable to change the uid err=%s\n", be.bstrerror()); + } else { + fprintf(stderr, "UID set! %d:%d\n", (int)getuid(), (int)getgid()); + sleep(5); + mkdir("/tmp/adirectory2", 0755); + system("touch /tmp/afile2"); + system("id"); + fclose(fopen("/tmp/aaa2", "a")); + } + + return NULL; +} + void *th2(void *temp) { P(mutex2); @@ -1350,6 +1436,10 @@ int report() return err>0; } +void terminate(int sig) +{ +} + /* * TODO: * - Must detect multiple lock @@ -1365,7 +1455,7 @@ int main(int argc, char **argv) pthread_mutex_t pmutex2; debug_level = 10; my_prog = argv[0]; - + init_signals(terminate); use_undertaker = false; lmgr_init_thread(); self = lmgr_get_thread_info(); @@ -1378,6 +1468,10 @@ int main(int argc, char **argv) return 0; } + pthread_create(&id5, NULL, thuid, NULL); + pthread_join(id5, NULL); + fprintf(stderr, "UID %d:%d\n", (int)getuid(), (int)getgid()); + exit(0); pthread_mutex_init(&bmutex1, NULL); bthread_mutex_set_priority(&bmutex1, 10); @@ -1396,6 +1490,12 @@ int main(int argc, char **argv) lmgr_v(&mutex1.mutex); /* a bit dirty */ pthread_join(id1, NULL); + pthread_create(&id1, NULL, nolock, NULL); + sleep(2); + ok(bthread_kill(id1, SIGUSR2) == 0, "Kill existing thread"); + pthread_join(id1, NULL); + ok(bthread_kill(id1, SIGUSR2) == -1, "Kill non-existing thread"); + ok(bthread_kill(pthread_self(), SIGUSR2) == -1, "Kill self"); pthread_create(&id1, NULL, nolock, NULL); sleep(2); @@ -1530,7 +1630,6 @@ int main(int argc, char **argv) pthread_join(id4, NULL); pthread_join(id5, NULL); - // // pthread_create(&id3, NULL, th3, NULL); // diff --git a/bacula/src/lib/lockmgr.h b/bacula/src/lib/lockmgr.h index 3948649986..71911af9ee 100644 --- a/bacula/src/lib/lockmgr.h +++ b/bacula/src/lib/lockmgr.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -34,6 +34,11 @@ void lmgr_v(pthread_mutex_t *m); */ intptr_t bthread_get_thread_id(); +/* + * Set the Thread Id of the current thread to limit I/O operations + */ +int bthread_change_uid(uid_t uid, gid_t gid); + #ifdef USE_LOCKMGR typedef struct bthread_mutex_t @@ -118,6 +123,12 @@ void bthread_mutex_set_priority(bthread_mutex_t *m, int prio); */ void lmgr_init_thread(); +/* + * Know if the current thread is registred (used when we + * do not control thread creation) + */ +bool lmgr_thread_is_initialized(); + /* * Call this function at the end of the thread */ @@ -221,6 +232,7 @@ int bthread_kill(pthread_t thread, int sig, # define lmgr_add_event_p(c, u, f, l) # define lmgr_add_event(c, u) # define lmgr_dump() +# define lmgr_thread_is_initialized() (1) # define lmgr_init_thread() # define lmgr_cleanup_thread() # define lmgr_pre_lock(m, prio, f, l) @@ -247,4 +259,40 @@ int bthread_kill(pthread_t thread, int sig, # define bthread_cond_wait_p(w, x, y, z) pthread_cond_wait(w,x) #endif /* USE_LOCKMGR */ +/* a very basic lock_guard implementation : + * Lock_guard is mostly usefull to garanty mutex unlocking. Also, it's exception safe. + * usage example: + * void foobar() + * { + * lock_guard protector(m_mutex); // m_mutex is locked + * // the following section is protected until the function exits and/or returns + * + * if (case == TRUE) + * { + * return; // when returning, m_mutex is unlocked + * } + * . + * . + * . + * + * // when the method exits, m_mutex is unlocked. + * } + */ +class lock_guard +{ +public: + + pthread_mutex_t &m_mutex; /* the class keeps a reference on the mutex*/ + + explicit lock_guard(pthread_mutex_t &mutex) : m_mutex(mutex) + { + P(m_mutex); /* constructor locks the mutex*/ + } + + ~lock_guard() + { + V(m_mutex); /* destructor unlocks the mutex*/ + } +}; + #endif /* LOCKMGR_H */ diff --git a/bacula/src/lib/lz4.c b/bacula/src/lib/lz4.c new file mode 100644 index 0000000000..1e174016c3 --- /dev/null +++ b/bacula/src/lib/lz4.c @@ -0,0 +1,685 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +#include "config.h" + +/* +Note : this source file requires "lz4_encoder.h" +*/ + +//************************************** +// Tuning parameters +//************************************** +// MEMORY_USAGE : +// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +// Increasing memory usage improves compression ratio +// Reduced memory usage can improve speed, due to cache effect +// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache +#define MEMORY_USAGE 14 + +// HEAPMODE : +// Select how default compression function will allocate memory for its hash table, +// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)). +// Default allocation strategy is to use stack (HEAPMODE 0) +// Note : explicit functions *_stack* and *_heap* are unaffected by this setting +#define HEAPMODE 0 + +// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : +// This will provide a small boost to performance for big endian cpu, but the resulting compressed stream will be incompatible with little-endian CPU. +// You can set this option to 1 in situations where data will remain within closed environment +// This option is useless on Little_Endian CPU (such as x86) +//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 + +#ifdef HAVE_BIG_ENDIAN +#define LZ4_BIG_ENDIAN 1 +#else +#if defined(__sparc) || defined(__sparc__) \ + || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ + || defined(__hpux) || defined(__hppa) \ + || defined(_MIPSEB) || defined(__s390__) +#error "BIG Endian detected but not set" +#endif +#endif + +//************************************** +// CPU Feature Detection +//************************************** +// 32 or 64 bits ? +#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ + || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ + || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ + || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode +# define LZ4_ARCH64 1 +#else +# define LZ4_ARCH64 0 +#endif + + +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +#if defined(__ARM_FEATURE_UNALIGNED) +# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#endif + +// Define this parameter if your target system or compiler does not support hardware bit count +#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +//************************************** +// Compiler Options +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/* "restrict" is a known keyword */ +#elif !defined restrict +# define restrict // Disable restrict +#endif + + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef _MSC_VER // Visual Studio +# include // For Visual 2005 +# if LZ4_ARCH64 // 64-bit +# pragma intrinsic(_BitScanForward64) // For Visual 2005 +# pragma intrinsic(_BitScanReverse64) // For Visual 2005 +# else +# pragma intrinsic(_BitScanForward) // For Visual 2005 +# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# endif +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER +# define lz4_bswap16(x) _byteswap_ushort(x) +#else +# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) +#endif + +#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +//************************************** +// Includes +//************************************** +#include // for malloc +#include // for memset +#include "lz4.h" + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(push, 1) +#endif + +typedef struct _U16_S { U16 v; } _PACKED U16_S; +typedef struct _U32_S { U32 v; } _PACKED U32_S; +typedef struct _U64_S { U64 v; } _PACKED U64_S; + +#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A64(x) (((U64_S *)(x))->v) +#define A32(x) (((U32_S *)(x))->v) +#define A16(x) (((U16_S *)(x))->v) + + +//************************************** +// Constants +//************************************** +#define HASHTABLESIZE (1 << MEMORY_USAGE) + +#define MINMATCH 4 + +#define COPYLENGTH 8 +#define LASTLITERALS 5 +#define MFLIMIT (COPYLENGTH+MINMATCH) +#define MINLENGTH (MFLIMIT+1) + +#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1)) +#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data + +#define MAXD_LOG 16 +#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) + +#define ML_BITS 4 +#define ML_MASK ((1U<>3); + #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); + #else + int r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; + #endif +#else + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll(val) >> 3); + #else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; + #endif +#endif +} + +#else + +static inline int LZ4_NbCommonBytes (register U32 val) +{ +#if defined(LZ4_BIG_ENDIAN) +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); +# else + int r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif +#else +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz(val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif +#endif +} + +#endif + + + +//****************************** +// Compression functions +//****************************** + +/* +int LZ4_compress_stack( + const char* source, + char* dest, + int inputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest'. +Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize). +return : the number of bytes written in buffer 'dest' +*/ +#define FUNCTION_NAME LZ4_compress_stack +#include "lz4_encoder.h" + + +/* +int LZ4_compress_stack_limitedOutput( + const char* source, + char* dest, + int inputSize, + int maxOutputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. +If it cannot achieve it, compression will stop, and result of the function will be zero. +return : the number of bytes written in buffer 'dest', or 0 if the compression fails +*/ +#define FUNCTION_NAME LZ4_compress_stack_limitedOutput +#define LIMITED_OUTPUT +#include "lz4_encoder.h" + + +/* +int LZ4_compress64k_stack( + const char* source, + char* dest, + int inputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest'. +This function compresses better than LZ4_compress_stack(), on the condition that +'inputSize' must be < to LZ4_64KLIMIT, or the function will fail. +Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize). +return : the number of bytes written in buffer 'dest', or 0 if compression fails +*/ +#define FUNCTION_NAME LZ4_compress64k_stack +#define COMPRESS_64K +#include "lz4_encoder.h" + + +/* +int LZ4_compress64k_stack_limitedOutput( + const char* source, + char* dest, + int inputSize, + int maxOutputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. +This function compresses better than LZ4_compress_stack_limitedOutput(), on the condition that +'inputSize' must be < to LZ4_64KLIMIT, or the function will fail. +If it cannot achieve it, compression will stop, and result of the function will be zero. +return : the number of bytes written in buffer 'dest', or 0 if the compression fails +*/ +#define FUNCTION_NAME LZ4_compress64k_stack_limitedOutput +#define COMPRESS_64K +#define LIMITED_OUTPUT +#include "lz4_encoder.h" + + +/* +void* LZ4_createHeapMemory(); +int LZ4_freeHeapMemory(void* ctx); + +Used to allocate and free hashTable memory +to be used by the LZ4_compress_heap* family of functions. +LZ4_createHeapMemory() returns NULL is memory allocation fails. +*/ +void* LZ4_create() { return malloc(HASHTABLESIZE); } +int LZ4_free(void* ctx) { free(ctx); return 0; } + + +/* +int LZ4_compress_heap( + void* ctx, + const char* source, + char* dest, + int inputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest'. +The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'. +Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize). +return : the number of bytes written in buffer 'dest' +*/ +#define FUNCTION_NAME LZ4_compress_heap +#define USE_HEAPMEMORY +#include "lz4_encoder.h" + + +/* +int LZ4_compress_heap_limitedOutput( + void* ctx, + const char* source, + char* dest, + int inputSize, + int maxOutputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. +If it cannot achieve it, compression will stop, and result of the function will be zero. +The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'. +return : the number of bytes written in buffer 'dest', or 0 if the compression fails +*/ +#define FUNCTION_NAME LZ4_compress_heap_limitedOutput +#define LIMITED_OUTPUT +#define USE_HEAPMEMORY +#include "lz4_encoder.h" + + +/* +int LZ4_compress64k_heap( + void* ctx, + const char* source, + char* dest, + int inputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest'. +The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'. +'inputSize' must be < to LZ4_64KLIMIT, or the function will fail. +Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize). +return : the number of bytes written in buffer 'dest' +*/ +#define FUNCTION_NAME LZ4_compress64k_heap +#define COMPRESS_64K +#define USE_HEAPMEMORY +#include "lz4_encoder.h" + + +/* +int LZ4_compress64k_heap_limitedOutput( + void* ctx, + const char* source, + char* dest, + int inputSize, + int maxOutputSize) + +Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. +If it cannot achieve it, compression will stop, and result of the function will be zero. +The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'. +'inputSize' must be < to LZ4_64KLIMIT, or the function will fail. +return : the number of bytes written in buffer 'dest', or 0 if the compression fails +*/ +#define FUNCTION_NAME LZ4_compress64k_heap_limitedOutput +#define COMPRESS_64K +#define LIMITED_OUTPUT +#define USE_HEAPMEMORY +#include "lz4_encoder.h" + + +int LZ4_compress(const char* source, char* dest, int inputSize) +{ +#if HEAPMODE + void* ctx = LZ4_create(); + int result; + if (ctx == NULL) return 0; // Failed allocation => compression not done + if (inputSize < LZ4_64KLIMIT) + result = LZ4_compress64k_heap(ctx, source, dest, inputSize); + else result = LZ4_compress_heap(ctx, source, dest, inputSize); + LZ4_free(ctx); + return result; +#else + if (inputSize < (int)LZ4_64KLIMIT) return LZ4_compress64k_stack(source, dest, inputSize); + return LZ4_compress_stack(source, dest, inputSize); +#endif +} + + +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ +#if HEAPMODE + void* ctx = LZ4_create(); + int result; + if (ctx == NULL) return 0; // Failed allocation => compression not done + if (inputSize < LZ4_64KLIMIT) + result = LZ4_compress64k_heap_limitedOutput(ctx, source, dest, inputSize, maxOutputSize); + else result = LZ4_compress_heap_limitedOutput(ctx, source, dest, inputSize, maxOutputSize); + LZ4_free(ctx); + return result; +#else + if (inputSize < (int)LZ4_64KLIMIT) return LZ4_compress64k_stack_limitedOutput(source, dest, inputSize, maxOutputSize); + return LZ4_compress_stack_limitedOutput(source, dest, inputSize, maxOutputSize); +#endif +} + + +//**************************** +// Decompression functions +//**************************** + +typedef enum { noPrefix = 0, withPrefix = 1 } prefix64k_directive; +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } end_directive; +typedef enum { full = 0, partial = 1 } exit_directive; + + +// This generic decompression function cover all use cases. +// It shall be instanciated several times, using different sets of directives +// Note that it is essential this generic function is really inlined, +// in order to remove useless branches during compilation optimisation. +static inline int LZ4_decompress_generic( + const char* source, + char* dest, + int inputSize, // + int outputSize, // OutputSize must be != 0; if endOnInput==endOnInputSize, this value is the max size of Output Buffer. + + int endOnInput, // endOnOutputSize, endOnInputSize + int prefix64k, // noPrefix, withPrefix + int partialDecoding, // full, partial + int targetOutputSize // only used if partialDecoding==partial + ) +{ + // Local Variables + const BYTE* restrict ip = (const BYTE*) source; + const BYTE* ref; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + + size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; +#if LZ4_ARCH64 + size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; +#endif + + + // Special case + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too large, better decode everything + if unlikely(outputSize==0) goto _output_error; // Empty output buffer + + + // Main Loop + while (1) + { + unsigned token; + size_t length; + + // get runlength + token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) + { + unsigned s=255; + while (((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-COPYLENGTH))) + { + if (partialDecoding) + { + if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer + if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer + } + else + { + if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there, due to parsing restrictions + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : not enough place for another match (min 4) + 5 literals + } + memcpy(op, ip, length); + ip += length; + op += length; + break; // Necessarily EOF, due to parsing restrictions + } + LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy; + + // get offset + LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; + if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer + + // get matchlength + if ((length=(token&ML_MASK)) == ML_MASK) + { + while (endOnInput ? ipoend-(COPYLENGTH)-(STEPSIZE-4)) + { + if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals + LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH)); + while(op> ((MINMATCH*8)-HASHLOG)) +#define LZ4_HASHVALUE(p) LZ4_HASH(A32(p)) + + + +//**************************** +// Function code +//**************************** + +int FUNCTION_NAME( +#ifdef USE_HEAPMEMORY + void* ctx, +#endif + const char* source, + char* dest, + int inputSize +#ifdef LIMITED_OUTPUT + ,int maxOutputSize +#endif + ) +{ +#ifdef USE_HEAPMEMORY + CURRENT_H_TYPE* HashTable = (CURRENT_H_TYPE*)ctx; +#else + CURRENT_H_TYPE HashTable[HASHTABLE_NBCELLS] = {0}; +#endif + + const BYTE* ip = (BYTE*) source; + CURRENTBASE(base); + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; +#define matchlimit (iend - LASTLITERALS) + + BYTE* op = (BYTE*) dest; +#ifdef LIMITED_OUTPUT + BYTE* const oend = op + maxOutputSize; +#endif + + int length; + const int skipStrength = SKIPSTRENGTH; + U32 forwardH; + + + // Init + if (inputSizeLZ4_64KLIMIT) return 0; // Size too large (not within 64K limit) +#endif +#ifdef USE_HEAPMEMORY + memset((void*)HashTable, 0, HASHTABLESIZE); +#endif + + // First Byte + HashTable[LZ4_HASHVALUE(ip)] = (CURRENT_H_TYPE)(ip - base); + ip++; forwardH = LZ4_HASHVALUE(ip); + + // Main Loop + for ( ; ; ) + { + int findMatchAttempts = (1U << skipStrength) + 3; + const BYTE* forwardIp = ip; + const BYTE* ref; + BYTE* token; + + // Find a match + do { + U32 h = forwardH; + int step = findMatchAttempts++ >> skipStrength; + ip = forwardIp; + forwardIp = ip + step; + + if unlikely(forwardIp > mflimit) { goto _last_literals; } + + forwardH = LZ4_HASHVALUE(forwardIp); + ref = base + HashTable[h]; + HashTable[h] = (CURRENT_H_TYPE)(ip - base); + + } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip))); + + // Catch up + while ((ip>anchor) && (ref>(BYTE*)source) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; } + + // Encode Literal length + length = (int)(ip - anchor); + token = op++; +#ifdef LIMITED_OUTPUT + if unlikely(op + length + (2 + 1 + LASTLITERALS) + (length>>8) > oend) return 0; // Check output limit +#endif + if (length>=(int)RUN_MASK) + { + int len = length-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(length<>8) > oend) return 0; // Check output limit +#endif + if (length>=(int)ML_MASK) + { + *token += ML_MASK; + length -= ML_MASK; + for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; } + if (length >= 255) { length-=255; *op++ = 255; } + *op++ = (BYTE)length; + } + else *token += (BYTE)length; + + // Test end of chunk + if (ip > mflimit) { anchor = ip; break; } + + // Fill table + HashTable[LZ4_HASHVALUE(ip-2)] = (CURRENT_H_TYPE)(ip - 2 - base); + + // Test next position + ref = base + HashTable[LZ4_HASHVALUE(ip)]; + HashTable[LZ4_HASHVALUE(ip)] = (CURRENT_H_TYPE)(ip - base); + if ((ref >= ip - MAX_DISTANCE) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } + + // Prepare next loop + anchor = ip++; + forwardH = LZ4_HASHVALUE(ip); + } + +_last_literals: + // Encode Last Literals + { + int lastRun = (int)(iend - anchor); +#ifdef LIMITED_OUTPUT + if (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) return 0; // Check output limit +#endif + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<JobStatus == JS_Terminated || jcr->JobStatus == JS_Warnings); + if (d->dest_code == MD_MAIL_ON_ERROR && success) { goto rem_temp_file; /* no mail */ } else if (d->dest_code == MD_MAIL_ON_SUCCESS && !success) { @@ -741,7 +744,7 @@ void term_msg() static bool open_dest_file(JCR *jcr, DEST *d, const char *mode) { - d->fd = fopen(d->where, mode); + d->fd = bfopen(d->where, mode); if (!d->fd) { berrno be; delivery_error(_("fopen %s failed: ERR=%s\n"), d->where, be.bstrerror()); @@ -784,6 +787,7 @@ void dispatch_message(JCR *jcr, int type, utime_t mtime, char *msg) MSGS *msgs; BPIPE *bpipe; const char *mode; + bool created_jcr = false; Dmsg2(850, "Enter dispatch_msg type=%d msg=%s", type, msg); @@ -829,6 +833,10 @@ void dispatch_message(JCR *jcr, int type, utime_t mtime, char *msg) if (!jcr) { jcr = get_jcr_from_tsd(); } + if (!jcr) { + jcr = new_jcr(sizeof(JCR), NULL); + created_jcr = true; + } if (jcr) { msgs = jcr->jcr_msgs; } @@ -847,6 +855,7 @@ void dispatch_message(JCR *jcr, int type, utime_t mtime, char *msg) return; } + for (d=msgs->dest_chain; d; d=d->next) { if (bit_is_set(type, d->msg_types)) { bool ok; @@ -879,7 +888,7 @@ void dispatch_message(JCR *jcr, int type, utime_t mtime, char *msg) case MD_CONSOLE: Dmsg1(850, "CONSOLE for following msg: %s", msg); if (!con_fd) { - con_fd = fopen(con_fname, "a+b"); + con_fd = bfopen(con_fname, "a+b"); Dmsg0(850, "Console file not open.\n"); } if (con_fd) { @@ -939,7 +948,7 @@ void dispatch_message(JCR *jcr, int type, utime_t mtime, char *msg) if (!d->fd) { POOLMEM *name = get_pool_memory(PM_MESSAGE); make_unique_mail_filename(jcr, name, d); - d->fd = fopen(name, "w+b"); + d->fd = bfopen(name, "w+b"); if (!d->fd) { berrno be; delivery_error(_("Msg delivery error: fopen %s failed: ERR=%s\n"), name, @@ -990,8 +999,8 @@ send_to_file: case MD_DIRECTOR: Dmsg1(850, "DIRECTOR for following msg: %s", msg); if (jcr && jcr->dir_bsock && !jcr->dir_bsock->errors) { - jcr->dir_bsock->fsend("Jmsg Job=%s type=%d level=%lld %s", - jcr->Job, type, mtime, msg); + jcr->dir_bsock->fsend("Jmsg JobId=%ld type=%d level=%lld %s", + jcr->JobId, type, mtime, msg); } else { Dmsg1(800, "no jcr for following msg: %s", msg); } @@ -1015,6 +1024,9 @@ send_to_file: } } } + if (created_jcr) { + free_jcr(jcr); + } } /********************************************************************* @@ -1051,7 +1063,7 @@ static void pt_out(char *buf) if (!trace_fd) { char fn[200]; bsnprintf(fn, sizeof(fn), "%s/%s.trace", working_directory ? working_directory : "./", my_name); - trace_fd = fopen(fn, "a+b"); + trace_fd = bfopen(fn, "a+b"); } if (trace_fd) { fputs(buf, trace_fd); @@ -1149,9 +1161,7 @@ void set_trace(int trace_flag) void set_hangup(int hangup_value) { - if (hangup_value < 0) { - return; - } else { + if (hangup_value != -1) { hangup = hangup_value; } } @@ -1163,9 +1173,7 @@ int get_hangup(void) void set_blowup(int blowup_value) { - if (blowup_value < 0) { - return; - } else { + if (blowup_value != -1) { blowup = blowup_value; } } @@ -1175,6 +1183,36 @@ int get_blowup(void) return blowup; } +bool handle_hangup_blowup(JCR *jcr, uint32_t file_count, uint64_t byte_count) +{ + if (hangup == 0 && blowup == 0) { + /* quick check */ + return false; + } + /* Debug code: check if we must hangup or blowup */ + if ((hangup > 0 && (file_count > (uint32_t)hangup)) || + (hangup < 0 && (byte_count/1024 > (uint32_t)-hangup))) { + jcr->setJobStatus(JS_Incomplete); + if (hangup > 0) { + Jmsg1(jcr, M_FATAL, 0, "Debug hangup requested after %d files.\n", hangup); + } else { + Jmsg1(jcr, M_FATAL, 0, "Debug hangup requested after %d Kbytes.\n", -hangup); + } + set_hangup(0); + return true; + } + if ((blowup > 0 && (file_count > (uint32_t)blowup)) || + (blowup < 0 && (byte_count/1024 > (uint32_t)-blowup))) { + if (blowup > 0) { + Jmsg1(jcr, M_ABORT, 0, "Debug blowup requested after %d files.\n", blowup); + } else { + Jmsg1(jcr, M_ABORT, 0, "Debug blowup requested after %d Kbytes.\n", -blowup); + } + /* will never reach this line */ + return true; + } + return false; +} bool get_trace(void) { @@ -1245,7 +1283,7 @@ t_msg(const char *file, int line, int64_t level, const char *fmt,...) if (level <= debug_level) { if (!trace_fd) { bsnprintf(buf, sizeof(buf), "%s/%s.trace", working_directory ? working_directory : ".", my_name); - trace_fd = fopen(buf, "a+b"); + trace_fd = bfopen(buf, "a+b"); } #ifdef FULL_LOCATION @@ -1327,7 +1365,6 @@ e_msg(const char *file, int line, int type, int level, const char *fmt,...) dispatch_message(NULL, type, 0, buf); if (type == M_ABORT) { - assert_msg = bstrdup(buf); char *p = 0; p[0] = 0; /* generate segmentation violation */ } @@ -1336,6 +1373,23 @@ e_msg(const char *file, int line, int type, int level, const char *fmt,...) } } +/* Check in the msgs resource if a given type is defined */ +bool is_message_type_set(JCR *jcr, int type) +{ + MSGS *msgs = NULL; + if (jcr) { + msgs = jcr->jcr_msgs; + } + if (!msgs) { + msgs = daemon_msgs; /* if no jcr, we use daemon handler */ + } + if (msgs && (type != M_ABORT && type != M_ERROR_TERM) && + !bit_is_set(type, msgs->send_msg)) { + return false; /* no destination */ + } + return true; +} + /* ********************************************************* * * Generate a Job message @@ -1447,7 +1501,6 @@ Jmsg(JCR *jcr, int type, utime_t mtime, const char *fmt,...) char *p = 0; printf("Bacula forced SEG FAULT to obtain traceback.\n"); syslog(LOG_DAEMON|LOG_ERR, "Bacula forced SEG FAULT to obtain traceback.\n"); - assert_msg = bstrdup(rbuf); p[0] = 0; /* generate segmentation violation */ } if (type == M_ERROR_TERM) { @@ -1633,14 +1686,17 @@ void Qmsg(JCR *jcr, int type, utime_t mtime, const char *fmt,...) } if (jcr && type==M_FATAL) { - // TODO ASX MUST use a lock to protect access jcr->JobStatus from another thread jcr->setJobStatus(JS_FatalError); } /* If no jcr or no queue or dequeuing send to syslog */ if (!jcr || !jcr->msg_queue || jcr->dequeuing_msgs) { syslog(LOG_DAEMON|LOG_ERR, "%s", item->msg); - free(item); + P(daemon_msg_queue_mutex); + if (daemon_msg_queue) { + daemon_msg_queue->append(item); + } + V(daemon_msg_queue_mutex); } else { /* Queue message for later sending */ P(jcr->msg_queue_mutex); @@ -1656,6 +1712,30 @@ void Qmsg(JCR *jcr, int type, utime_t mtime, const char *fmt,...) void dequeue_messages(JCR *jcr) { MQUEUE_ITEM *item; + JobId_t JobId; + + /* Avoid bad calls and recursion */ + if (jcr == NULL || jcr->dequeuing_msgs) { + return; + } + + /* Dequeue daemon messages */ + if (daemon_msg_queue) { + P(daemon_msg_queue_mutex); + jcr->dequeuing_msgs = true; + JobId = jcr->JobId; + jcr->JobId = 0; /* set daemon JobId == 0 */ + foreach_dlist(item, daemon_msg_queue) { + Jmsg(jcr, item->type, item->mtime, "%s", item->msg); + } + /* Remove messages just sent */ + daemon_msg_queue->destroy(); + jcr->JobId = JobId; /* restore JobId */ + jcr->dequeuing_msgs = false; + V(daemon_msg_queue_mutex); + } + + /* Dequeue Job specific messages */ if (!jcr->msg_queue) { return; } diff --git a/bacula/src/lib/message.h b/bacula/src/lib/message.h index 365ec2461b..075f9e3774 100644 --- a/bacula/src/lib/message.h +++ b/bacula/src/lib/message.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -69,6 +69,8 @@ * M_ALERT For Tape Alert messages. * * M_VOLMGMT Volume Management message + * + * M_DEBUG and M_SAVED are excluded from M_ALL by default */ enum { @@ -156,6 +158,8 @@ struct MQUEUE_ITEM { #define DT_MEMORY (1<<24) /* memory */ #define DT_SCHEDULER (1<<23) /* scheduler */ #define DT_PROTOCOL (1<<22) /* protocol */ +#define DT_xxxxx (1<<21) /* reserved BEE */ +#define DT_xxx (1<<20) /* reserved BEE */ #define DT_SNAPSHOT (1<<19) /* Snapshot */ #define DT_ASX (1<<16) /* used by Alain for personal debugging */ #define DT_ALL (0x7FFF0000) /* all (up to debug_level 65635, 15 flags available) */ @@ -172,6 +176,7 @@ void Qmsg(JCR *jcr, int type, utime_t mtime, const char *fmt,...) CHECK_FORMAT(p bool get_trace(void); void set_debug_flags(char *options); const char *get_basename(const char *pathname); +bool is_message_type_set(JCR *jcr, int type); class BDB; /* define forward reference */ typedef bool (*sql_query_call)(JCR *jcr, const char *cmd); diff --git a/bacula/src/lib/output.c b/bacula/src/lib/output.c new file mode 100644 index 0000000000..a893106892 --- /dev/null +++ b/bacula/src/lib/output.c @@ -0,0 +1,496 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Written by: Eric Bollengier, December MMXIII + */ + +#define OUTPUT_C /* control dll export in output.h */ +#include "output.h" +#include "plugins.h" + +/* use new output (lowercase, no special char) */ +#define OF_USE_NEW_OUTPUT 1 + +void OutputWriter::parse_options(const char *options) +{ + int nb=0; + const char *p = options; + while (*p) { + nb=0; + + switch(*p) { + case 'C': + flags = 0; + set_time_format(OW_DEFAULT_TIMEFORMAT); + set_separator(OW_DEFAULT_SEPARATOR); + break; + + case 'S': /* object separator */ + while(isdigit(*(p+1))) { + nb = nb*10 + (*(++p) - '0'); + } + if (isascii(nb)) { + set_object_separator((char) nb); + } + break; + + case 'o': + flags |= OF_USE_NEW_OUTPUT; /* lowercase and only isalpha */ + break; + + case 't': /* Time format */ + if (isdigit(*(p+1))) { + nb = (*(++p) - '0'); + set_time_format((OutputTimeType) nb); + } + break; + + case 's': /* Separator */ + while(isdigit(*(p+1))) { + nb = nb*10 + (*(++p) - '0'); + } + if (isascii(nb)) { + set_separator((char) nb); + } + break; + default: + break; + } + p++; + } +} + +char *OutputWriter::get_options(char *dest) +{ + char ed1[50]; + *dest = *ed1 = 0; + if (separator != OW_DEFAULT_SEPARATOR) { + snprintf(dest, 50, "s%d", (int)separator); + } + if (object_separator) { + snprintf(ed1, sizeof(ed1), "S%d", (int) object_separator); + bstrncat(dest, ed1, sizeof(ed1)); + } + if (timeformat != OW_DEFAULT_TIMEFORMAT) { + snprintf(ed1, sizeof(ed1), "t%d", (int) timeformat); + bstrncat(dest, ed1, sizeof(ed1)); + } + if (flags & OF_USE_NEW_OUTPUT) { + bstrncat(dest, "o", 1); + } + return dest; +} + +void OutputWriter::get_buf(bool append) +{ + if (!buf) { + buf = get_pool_memory(PM_MESSAGE); + *buf = 0; + + } else if (!append) { + *buf = 0; + } +} + +char *OutputWriter::start_group(const char *name, bool append) +{ + get_buf(append); + pm_strcat(buf, name); + pm_strcat(buf, ":\n"); + return buf; +} + +char *OutputWriter::end_group(bool append) +{ + get_buf(append); + pm_strcat(buf, "\n"); + + return buf; +} + +char *OutputWriter::start_list(const char *name, bool append) +{ + get_buf(append); + pm_strcat(buf, name); + pm_strcat(buf, ": [\n"); + return buf; +} + +char *OutputWriter::end_list(bool append) +{ + get_buf(append); + pm_strcat(buf, "]\n"); + + return buf; +} + +/* Usage: + * get_output( + * OT_STRING, "name", "value", + * OT_PINT32, "age", 10, + * OT_TIME, "birth-date", 1120202002, + * OT_PINT64, "weight", 100, + * OT_END); + * + * + * "name=value\nage=10\nbirt-date=2012-01-12 10:20:00\nweight=100\n" + * + */ +char *OutputWriter::get_output(OutputType first, ...) +{ + char *ret; + va_list arg_ptr; + + get_buf(true); /* Append to the current string */ + + va_start(arg_ptr, first); + ret = get_output(arg_ptr, &buf, first); + va_end(arg_ptr); + + return ret; +} + +/* Usage: + * get_output(&out, + * OT_STRING, "name", "value", + * OT_PINT32, "age", 10, + * OT_TIME, "birth-date", 1120202002, + * OT_PINT64, "weight", 100, + * OT_END); + * + * + * "name=value\nage=10\nbirt-date=2012-01-12 10:20:00\nweight=100\n" + * + */ +char *OutputWriter::get_output(POOLMEM **out, OutputType first, ...) +{ + va_list arg_ptr; + char *ret; + + va_start(arg_ptr, first); + ret = get_output(arg_ptr, out, first); + va_end(arg_ptr); + + return ret; +} + +char *OutputWriter::get_output(va_list ap, POOLMEM **out, OutputType first) +{ + char ed1[MAX_TIME_LENGTH]; + int i; + int64_t i64; + uint64_t u64; + int32_t i32; + double d; + btime_t bt; + char *s = NULL, *k = NULL; + alist *lst; + Plugin *plug; + POOLMEM *tmp2 = get_pool_memory(PM_FNAME); + POOLMEM *tmp = get_pool_memory(PM_FNAME); + OutputType val = first; + + while (val != OT_END) { + + *tmp = 0; + + /* Some arguments are not using a keyword */ + switch (val) { + case OT_END: + case OT_START_OBJ: + case OT_END_OBJ: + case OT_CLEAR: + break; + + default: + k = va_arg(ap, char *); /* Get the variable name */ + + /* If requested, we can put the keyword in lowercase */ + if (flags & OF_USE_NEW_OUTPUT) { + tmp2 = check_pool_memory_size(tmp2, strlen(k)+1); + for (i = 0; k[i] ; i++) { + if (isalnum(k[i])) { + tmp2[i] = tolower(k[i]); + } else { + tmp2[i] = '_'; + } + } + tmp2[i] = 0; + k = tmp2; + } + } + + //Dmsg2(000, "%d - %s\n", val, k); + + switch (val) { + case OT_ALIST_STR: + lst = va_arg(ap, alist *); + i = 0; + Mmsg(tmp, "%s=", k); + if (lst) { + foreach_alist(s, lst) { + if (i++ > 0) { + pm_strcat(tmp, ","); + } + pm_strcat(tmp, s); + } + } + pm_strcat(tmp, separator_str); + break; + case OT_PLUGINS: + lst = va_arg(ap, alist *); + i = 0; + pm_strcpy(tmp, "plugins="); + if (lst) { + foreach_alist(plug, lst) { + if (i++ > 0) { + pm_strcat(tmp, ","); + } + pm_strcat(tmp, plug->file); + } + } + pm_strcat(tmp, separator_str); + break; + case OT_RATIO: + d = va_arg(ap, double); + Mmsg(tmp, "%s=%.2f%c", k, d, separator); + break; + + case OT_STRING: + s = va_arg(ap, char *); + Mmsg(tmp, "%s=%s%c", k, NPRTB(s), separator) ; + break; + + case OT_INT32: + i32 = va_arg(ap, int32_t); + Mmsg(tmp, "%s=%d%c", k, i32, separator); + break; + + case OT_UTIME: + case OT_BTIME: + if (val == OT_UTIME) { + bt = va_arg(ap, utime_t); + } else { + bt = va_arg(ap, btime_t); + } + switch (timeformat) { + case OTT_TIME_NC: /* Formatted time for user display: dd-Mon hh:mm */ + bstrftime_ny(ed1, sizeof(ed1), bt); + break; + + case OTT_TIME_UNIX: /* unix timestamp */ + bsnprintf(ed1, sizeof(ed1), "%lld", bt); + break; + + case OTT_TIME_ISO: + /* wanted fallback */ + default: + bstrutime(ed1, sizeof(ed1), bt); + } + Mmsg(tmp, "%s_epoch=%lld%c%s=%s%c", k, bt, separator, k, ed1, separator); + break; + + case OT_SIZE: + case OT_INT64: + i64 = va_arg(ap, int64_t); + Mmsg(tmp, "%s=%lld%c", k, i64, separator); + break; + + case OT_PINT64: + u64 = va_arg(ap, uint64_t); + Mmsg(tmp, "%s=%llu%c", k, u64, separator); + break; + + case OT_INT: + i64 = va_arg(ap, int); + Mmsg(tmp, "%s=%lld%c", k, i64, separator); + break; + + case OT_JOBLEVEL: + case OT_JOBTYPE: + case OT_JOBSTATUS: + i32 = va_arg(ap, int32_t); + Mmsg(tmp, "%s=%c%c", k, (char) i32, separator); + break; + + case OT_CLEAR: + **out = 0; + break; + + case OT_END_OBJ: + pm_strcpy(tmp, "\n"); + break; + + case OT_START_OBJ: + i=0; + if (object_separator) { + for(; i < 32 ; i++) { + tmp[i] = object_separator; + } + } + tmp[i++] = '\n'; + tmp[i] = 0; + break; + + case OT_END: + /* wanted fallback */ + default: + val = OT_END; + } + + if (val != OT_END) { + pm_strcat(out, tmp); + val = (OutputType) va_arg(ap, int); /* OutputType is promoted to int when using ... */ + } + } + + free_pool_memory(tmp); + free_pool_memory(tmp2); + //Dmsg1(000, "%s", *out); + return *out; +} + +#ifdef TEST_PROGRAM +int err=0; +int nb=0; +void _ok(const char *file, int l, const char *op, int value, const char *label) +{ + nb++; + if (!value) { + err++; + printf("ERR %.30s %s:%i on %s\n", label, file, l, op); + } else { + printf("OK %.30s\n", label); + } +} + +#define ok(x, label) _ok(__FILE__, __LINE__, #x, (x), label) + +void _nok(const char *file, int l, const char *op, int value, const char *label) +{ + nb++; + if (value) { + err++; + printf("ERR %.30s %s:%i on !%s\n", label, file, l, op); + } else { + printf("OK %.30s\n", label); + } +} + +#define nok(x, label) _nok(__FILE__, __LINE__, #x, (x), label) + +int report() +{ + printf("Result %i/%i OK\n", nb - err, nb); + return err>0; +} + +int main(int argc, char **argv) +{ + char ed1[50]; + OutputWriter wt; + POOLMEM *tmp = get_pool_memory(PM_FNAME); + *tmp = 0; + + int nb = 10000; + const char *ptr = "my value"; + char *str = bstrdup("ptr"); + int32_t nb32 = -1; + int64_t nb64 = -1; + btime_t t = time(NULL); + + ok(strcmp(wt.get_options(ed1), "") == 0, "Default options"); + + Pmsg1(000, "%s", wt.start_group("test")); + + wt.get_output(&tmp, OT_CLEAR, + OT_STRING, "test", "my value", + OT_STRING, "test2", ptr, + OT_STRING, "test3", str, + OT_INT, "nb", nb, + OT_INT32, "nb32", nb32, + OT_INT64, "nb64", nb64, + OT_BTIME, "now", t, + OT_END); + + Pmsg1(000, "%s", tmp); + + free_pool_memory(tmp); + + + Pmsg1(000, "%s", + wt.get_output(OT_CLEAR, + OT_START_OBJ, + OT_STRING, "test", "my value", + OT_STRING, "test2", ptr, + OT_STRING, "test3", str, + OT_INT, "nb", nb, + OT_INT32, "nb32", nb32, + OT_INT64, "nb64", nb64, + OT_BTIME, "now", t, + OT_END_OBJ, + OT_END)); + + wt.set_time_format(OTT_TIME_UNIX); + ok(strcmp("t1", wt.get_options(ed1)) == 0, "Check unix time format"); + + Pmsg1(000, "%s", + wt.get_output(OT_CLEAR, + OT_BTIME, "now", t, + OT_END)); + + wt.set_time_format(OTT_TIME_NC); + ok(strcmp("t2", wt.get_options(ed1)) == 0, "Check NC time format"); + + Pmsg1(000, "%s", + wt.get_output(OT_CLEAR, + OT_BTIME, "now", t, + OT_END)); + + Pmsg1(000, "%s", wt.end_group(false)); + + wt.parse_options("s43t1O"); + ok(strcmp(wt.get_options(ed1), "s43t1") == 0, "Check options after parsing"); + + ok(strstr( + wt.get_output(OT_CLEAR, + OT_BTIME, "now", t, + OT_STRING, "brazil", "test", + OT_END), + "+brazil=test+") != NULL, + "Check separator"); + + wt.parse_options("CS35"); + ok(strcmp(wt.get_options(ed1), "S35") == 0, "Check options after parsing"); + + Pmsg1(000, "%s", + wt.get_output(OT_CLEAR, + OT_START_OBJ, + OT_STRING, "test", "my value", + OT_STRING, "test2", ptr, + OT_END_OBJ, + OT_START_OBJ, + OT_STRING, "test", "my value", + OT_STRING, "test2", ptr, + OT_END_OBJ, + OT_END)); + + return report(); +} +#endif diff --git a/bacula/src/lib/output.h b/bacula/src/lib/output.h new file mode 100644 index 0000000000..382cd29938 --- /dev/null +++ b/bacula/src/lib/output.h @@ -0,0 +1,159 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Written by: Eric Bollengier, December MMXIII + */ + +#ifndef OUTPUT_H +#define OUTPUT_H + +#include "bacula.h" + +typedef enum { + OT_INT, /* Integer */ + OT_SIZE, /* int64 size */ + OT_PINT32, /* Uint32 */ + OT_INT32, + OT_PINT64, /* Uint64 */ + OT_INT64, + OT_STRING, + OT_BTIME, /* btime_t */ + OT_UTIME, /* utime_t */ + OT_JOBTYPE, + OT_JOBLEVEL, + OT_JOBSTATUS, + OT_PLUGINS, /* Plugin alist */ + OT_RATIO, /* Double %.2f format */ + OT_ALIST_STR, + + OT_END, /* Last operator (no extra arg) */ + OT_START_OBJ, /* Skip a line to start a new object (no extra arg) */ + OT_END_OBJ, /* Skip a line to end current object (no extra arg) */ + OT_CLEAR /* truncate current buffer (no extra arg) */ +} OutputType; + +/* Keep the same order for get_options/parse_options */ +typedef enum { + OTT_TIME_ISO = 0, + OTT_TIME_UNIX = 1, /* unix time stamp */ + OTT_TIME_NC = 2 /* Formatted time for user display: dd-Mon hh:mm */ +} OutputTimeType; + +#define OW_DEFAULT_SEPARATOR '\n' +#define OW_DEFAULT_TIMEFORMAT OTT_TIME_ISO + +/* If included from output.c, mark the class as export (else, symboles are + * exported from all files... + */ +#ifdef OUTPUT_C +# define OUTPUT_EXPORT DLL_IMP_EXP +#else +# define OUTPUT_EXPORT +#endif + +class OUTPUT_EXPORT OutputWriter: public SMARTALLOC +{ +private: + void init() { + buf = NULL; + separator = OW_DEFAULT_SEPARATOR; + separator_str[0] = OW_DEFAULT_SEPARATOR; + separator_str[1] = 0; + timeformat = OW_DEFAULT_TIMEFORMAT; + object_separator = 0; + flags = 0; + }; + +protected: + virtual char *get_output(va_list ap, POOLMEM **out, OutputType first); + void get_buf(bool append); /* Allocate buf if needed */ + + int flags; + char separator; + char separator_str[2]; + char object_separator; + OutputTimeType timeformat; + POOLMEM *buf; + +public: + OutputWriter(const char *opts) { + init(); + parse_options(opts); + }; + + OutputWriter() { + init(); + }; + + virtual ~OutputWriter() { + free_and_null_pool_memory(buf); + }; + + /* s[ascii code]t[0-3] + * ^^^ ^^ + * separator time format + * "s43" => + will be used as separator + * "s43t1" => + as separator and time as unix timestamp + */ + virtual void parse_options(const char *opts); + virtual char *get_options(char *dest_l128); /* MAX_NAME_LENGTH mini */ + + /* Make a clear separation in the output*/ + virtual char *start_group(const char *name, bool append=true); + virtual char *end_group(bool append=true); + + /* Make a clear separation in the output for list*/ + virtual char *start_list(const char *name, bool append=true); + virtual char *end_list(bool append=true); + + /* \n by default, can be \t for example */ + void set_separator(char sep) { + separator = sep; + separator_str[0] = sep; + }; + + void set_object_separator(char sep) { + object_separator = sep; + }; + + void set_time_format(OutputTimeType fmt) { + timeformat = fmt; + }; + +/* Usage: + * get_output(&out, + * OT_STRING, "name", "value", + * OT_PINT32, "age", 10, + * OT_TIME, "birth-date", 1120202002, + * OT_PINT64, "weight", 100, + * OT_END); + * + * + * "name=value\nage=10\nbirt-date=2012-01-12 10:20:00\nweight=100\n" + * + */ + + /* Use a user supplied buffer */ + char *get_output(POOLMEM **out, OutputType first, ...); + + /* Use the internal buffer */ + char *get_output(OutputType first, ...); +}; + +#endif diff --git a/bacula/src/lib/parse_conf.c b/bacula/src/lib/parse_conf.c index 1da62d6306..cb3573fbdd 100644 --- a/bacula/src/lib/parse_conf.c +++ b/bacula/src/lib/parse_conf.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -81,11 +81,16 @@ extern brwlock_t res_lock; /* resource lock */ /* Forward referenced subroutines */ static void scan_types(LEX *lc, MSGS *msg, int dest, char *where, char *cmd); static const char *get_default_configdir(); -static bool find_config_file(const char *config_file, char *full_path, int max_path); /* Common Resource definitions */ -/* Message resource directives +/* + * Message resource directives + * Note: keep all store_mesgs last in the list as they are all + * output in json as a list. + * Also, the list store_msgs item must have flags set to ITEM_LAST + * so that the list editor (bjson.c) knows when to stop. + * * name handler value code flags default_value */ RES_ITEM msgs_items[] = { @@ -93,6 +98,7 @@ RES_ITEM msgs_items[] = { {"Description", store_str, ITEM(res_msgs.hdr.desc), 0, 0, 0}, {"MailCommand", store_str, ITEM(res_msgs.mail_cmd), 0, ITEM_ALLOW_DUPS, 0}, {"OperatorCommand", store_str, ITEM(res_msgs.operator_cmd), 0, ITEM_ALLOW_DUPS, 0}, + /* See comments above */ {"Syslog", store_msgs, ITEM(res_msgs), MD_SYSLOG, 0, 0}, {"Mail", store_msgs, ITEM(res_msgs), MD_MAIL, 0, 0}, {"MailOnError", store_msgs, ITEM(res_msgs), MD_MAIL_ON_ERROR, 0, 0}, @@ -108,19 +114,15 @@ RES_ITEM msgs_items[] = { {NULL, NULL, {0}, 0, 0, 0} }; -struct s_mtypes { - const char *name; - int token; -}; /* Various message types */ -static struct s_mtypes msg_types[] = { - {"Debug", M_DEBUG}, +s_kw msg_types[] = { + {"Debug", M_DEBUG}, /* Keep 1st place */ + {"Saved", M_SAVED}, /* Keep 2nd place */ {"Abort", M_ABORT}, {"Fatal", M_FATAL}, {"Error", M_ERROR}, {"Warning", M_WARNING}, {"Info", M_INFO}, - {"Saved", M_SAVED}, {"NotSaved", M_NOTSAVED}, {"Skipped", M_SKIPPED}, {"Mount", M_MOUNT}, @@ -134,6 +136,7 @@ static struct s_mtypes msg_types[] = { {NULL, 0} }; + /* * Tape Label types permitted in Pool records * @@ -166,6 +169,52 @@ const char *res_to_str(int rcode) } } +/* + * Create a new res_head pointer to a list of res_heads + */ +void CONFIG::init_res_head(RES_HEAD ***rhead, int32_t rfirst, int32_t rlast) +{ + int num = rlast - rfirst + 1; + RES *res = NULL; + RES_HEAD **rh; + rh = *rhead = (RES_HEAD **)malloc(num * sizeof(RES_HEAD)); + for (int i=0; ires_list = New(rblist(res, &res->link)); + rh[i]->first = NULL; + rh[i]->last = NULL; + } +} + +/* + * Insert the resource in res_all into the + * resource list. + */ +bool CONFIG::insert_res(int rindex, int size) +{ + RES *res; + rblist *list = m_res_head[rindex]->res_list; + res = (RES *)malloc(size); + memcpy(res, m_res_all, size); + if (list->empty()) { + list->insert(res, res_compare); + m_res_head[rindex]->first = res; + m_res_head[rindex]->last = res; + } else { + RES *item, *prev; + prev = m_res_head[rindex]->last; + item = (RES *)list->insert(res, res_compare); + if (item != res) { + Mmsg(m_errmsg, _("Attempt to define second \"%s\" resource named \"%s\" is not permitted.\n"), + resources[rindex].name, ((URES *)res)->hdr.name); + return false; + } + prev->res_next = res; + m_res_head[rindex]->last = res; + } + Dmsg2(900, _("Inserted res: %s index=%d\n"), ((URES *)res)->hdr.name, rindex); + return true; +} /* * Initialize the static structure to zeros, then @@ -175,7 +224,7 @@ static void init_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { int i; int rindex = type - r_first; - + memset(config->m_res_all, 0, config->m_res_all_size); res_all.hdr.rcode = type; res_all.hdr.refcnt = 1; @@ -213,6 +262,39 @@ static void init_resource(CONFIG *config, int type, RES_ITEM *items, int pass) } } +/* Initialize a resouce with default values */ +bool init_resource(CONFIG *config, uint32_t type, void *res) +{ + RES_ITEM *items; + for (int i=0; resources[i].name; i++) { + if (resources[i].rcode == type) { + items = resources[i].items; + if (!items) { + return false; + } + init_resource(config, type, items, 1); + memcpy(res, config->m_res_all, config->m_res_all_size); + return true; + } + } + return false; +} + +/* + * Dump each resource of type + */ +void dump_each_resource(int type, void sendit(void *sock, const char *fmt, ...), void *sock) +{ + RES *res = NULL; + + if (type < 0) { /* no recursion */ + type = -type; + } + foreach_res(res, type) { + dump_resource(-type, res, sendit, sock); + } +} + /* Store Messages Destination information */ void store_msgs(LEX *lc, RES_ITEM *item, int index, int pass) @@ -336,8 +418,8 @@ static void scan_types(LEX *lc, MSGS *msg, int dest_code, char *where, char *cmd } if (msg_type == M_MAX+1) { /* all? */ - for (i=1; i<=M_MAX; i++) { /* yes set all types */ - add_msg_dest(msg, dest_code, i, where, cmd); + for (i=2; i<=M_MAX; i++) { /* yes set all types except Debug and Saved */ + add_msg_dest(msg, dest_code, msg_types[i].token, where, cmd); } } else if (is_not) { rem_msg_dest(msg, dest_code, msg_type, where); @@ -361,6 +443,7 @@ static void scan_types(LEX *lc, MSGS *msg, int dest_code, char *where, char *cmd void store_name(LEX *lc, RES_ITEM *item, int index, int pass) { POOLMEM *msg = get_pool_memory(PM_EMSG); + lex_get_token(lc, T_NAME); if (!is_name_valid(lc->str, &msg)) { scan_err1(lc, "%s\n", msg); @@ -431,6 +514,14 @@ void store_dir(LEX *lc, RES_ITEM *item, int index, int pass) if (lc->str[0] != '|') { do_shell_expansion(lc->str, sizeof_pool_memory(lc->str)); } +#ifdef STANDARDIZED_DIRECTORY_USAGE + // TODO ASX we should store all directory without the ending slash to + // avoid the need of testing its presence + int len=strlen(lc->str); + if (len>0 && IsPathSeparator(lc->str[len-1])) { + lc->str[len-1]='\0'; + } +#endif if (*(item->value)) { scan_err5(lc, _("Attempt to redefine \"%s\" from \"%s\" to \"%s\" referenced on line %d : %s\n"), item->name, *(item->value), lc->str, lc->line_no, lc->line); @@ -451,25 +542,29 @@ void store_password(LEX *lc, RES_ITEM *item, int index, int pass) unsigned char digest[CRYPTO_DIGEST_MD5_SIZE]; char sig[100]; + if (lc->options & LOPT_NO_MD5) { + store_str(lc, item, index, pass); - lex_get_token(lc, T_STRING); - if (pass == 1) { - MD5Init(&md5c); - MD5Update(&md5c, (unsigned char *) (lc->str), lc->str_len); - MD5Final(digest, &md5c); - for (i = j = 0; i < sizeof(digest); i++) { - sprintf(&sig[j], "%02x", digest[i]); - j += 2; - } - if (*(item->value)) { - scan_err5(lc, _("Attempt to redefine \"%s\" from \"%s\" to \"%s\" referenced on line %d : %s\n"), - item->name, *(item->value), lc->str, lc->line_no, lc->line); - return; + } else { + lex_get_token(lc, T_STRING); + if (pass == 1) { + MD5Init(&md5c); + MD5Update(&md5c, (unsigned char *) (lc->str), lc->str_len); + MD5Final(digest, &md5c); + for (i = j = 0; i < sizeof(digest); i++) { + sprintf(&sig[j], "%02x", digest[i]); + j += 2; + } + if (*(item->value)) { + scan_err5(lc, _("Attempt to redefine \"%s\" from \"%s\" to \"%s\" referenced on line %d : %s\n"), + item->name, *(item->value), lc->str, lc->line_no, lc->line); + return; + } + *(item->value) = bstrdup(sig); } - *(item->value) = bstrdup(sig); + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); } - scan_to_eol(lc); - set_bit(index, res_all.hdr.item_present); } @@ -485,7 +580,7 @@ void store_res(LEX *lc, RES_ITEM *item, int index, int pass) if (pass == 2) { res = GetResWithName(item->code, lc->str); if (res == NULL) { - scan_err3(lc, _("Could not find config Resource %s referenced on line %d : %s\n"), + scan_err3(lc, _("Could not find config Resource \"%s\" referenced on line %d : %s\n"), lc->str, lc->line_no, lc->line); return; } @@ -538,7 +633,7 @@ void store_alist_res(LEX *lc, RES_ITEM *item, int index, int pass) res = GetResWithName(item->code, lc->str); if (res == NULL) { scan_err3(lc, _("Could not find config Resource \"%s\" referenced on line %d : %s\n"), - item->name, lc->line_no, lc->line); + lc->str, lc->line_no, lc->line); return; } Dmsg5(900, "Append %p to alist %p size=%d i=%d %s\n", @@ -546,6 +641,11 @@ void store_alist_res(LEX *lc, RES_ITEM *item, int index, int pass) list->append(res); (item->value)[i] = (char *)list; if (lc->ch != ',') { /* if no other item follows */ + if (!lex_check_eol(lc)) { + /* found garbage at the end of the line */ + scan_err3(lc, _("Found unexpected characters resource list in Directive \"%s\" at the end of line %d : %s\n"), + item->name, lc->line_no, lc->line); + } break; /* get out */ } lex_get_token(lc, T_ALL); /* eat comma */ @@ -576,6 +676,11 @@ void store_alist_str(LEX *lc, RES_ITEM *item, int index, int pass) lc->str, list, list->size(), item->name); list->append(bstrdup(lc->str)); if (lc->ch != ',') { /* if no other item follows */ + if (!lex_check_eol(lc)) { + /* found garbage at the end of the line */ + scan_err3(lc, _("Found unexpected characters in resource list in Directive \"%s\" at the end of line %d : %s\n"), + item->name, lc->line_no, lc->line); + } break; /* get out */ } lex_get_token(lc, T_ALL); /* eat comma */ @@ -648,7 +753,10 @@ enum store_unit_type { STORE_SPEED } ; -/* Store a size in bytes */ +/* + * This routine stores either a 32 or a 64 bit value (size32) + * and either a size (in bytes) or a speed (bytes per second). + */ static void store_int_unit(LEX *lc, RES_ITEM *item, int index, int pass, bool size32, enum store_unit_type type) { @@ -829,14 +937,6 @@ enum parse_state { p_resource }; -CONFIG *new_config_parser() -{ - CONFIG *config; - config = (CONFIG *)malloc(sizeof(CONFIG)); - memset(config, 0, sizeof(CONFIG)); - return config; -} - void CONFIG::init( const char *cf, LEX_ERROR_HANDLER *scan_error, @@ -846,7 +946,7 @@ void CONFIG::init( int32_t r_first, int32_t r_last, RES_TABLE *resources, - RES **res_head) + RES_HEAD ***res_head) { m_cf = cf; m_scan_error = scan_error; @@ -856,7 +956,8 @@ void CONFIG::init( m_r_first = r_first; m_r_last = r_last; m_resources = resources; - m_res_head = res_head; + init_res_head(res_head, r_first, r_last); + m_res_head = *res_head; } /********************************************************************* @@ -880,6 +981,7 @@ bool CONFIG::parse_config() const char *cf = m_cf; LEX_ERROR_HANDLER *scan_error = m_scan_error; int err_type = m_err_type; + //HPKT hpkt; if (first && (errstat=rwl_init(&res_lock)) != 0) { berrno be; @@ -921,6 +1023,9 @@ bool CONFIG::parse_config() free(lc); return 0; } + if (!m_encode_pass) { + lex_store_clear_passwords(lc); + } lex_set_error_handler_error_type(lc, err_type) ; while ((token=lex_get_token(lc, T_ALL)) != T_EOF) { Dmsg3(900, "parse state=%d pass=%d got token=%s\n", state, pass, @@ -968,6 +1073,12 @@ bool CONFIG::parse_config() goto bail_out; } for (i=0; items[i].name; i++) { + //hpkt.pass = pass; + //hpkt.ritem = &items[i]; + //hpkt.edbuf = NULL; + //hpkt.index = i; + //hpkt.lc = lc; + //hpkt.hfunc = HF_STORE; if (strcasecmp(items[i].name, lc->str) == 0) { /* If the ITEM_NO_EQUALS flag is set we do NOT * scan for = after the keyword */ @@ -1003,7 +1114,10 @@ bool CONFIG::parse_config() scan_err0(lc, _("Name not specified for resource")); goto bail_out; } - save_resource(res_type, items, pass); /* save resource */ + if (!save_resource(this, res_type, items, pass)) { /* save resource */ + scan_err1(lc, "%s", m_errmsg); + goto bail_out; + } break; case T_EOL: @@ -1024,10 +1138,10 @@ bool CONFIG::parse_config() scan_err0(lc, _("End of conf file reached with unclosed resource.")); goto bail_out; } - if (debug_level >= 900 && pass == 2) { + if (chk_dbglvl(900) && pass == 2) { int i; for (i=m_r_first; i<=m_r_last; i++) { - dump_resource(i, m_res_head[i-m_r_first], prtmsg, NULL); + dump_each_resource(i, prtmsg, NULL); } } lc = lex_close_file(lc); @@ -1046,11 +1160,30 @@ const char *get_default_configdir() return SYSCONFDIR; } +#ifdef xxx_not_used + HRESULT hr; + static char szConfigDir[MAX_PATH + 1] = { 0 }; + if (!p_SHGetFolderPath) { + bstrncpy(szConfigDir, DEFAULT_CONFIGDIR, sizeof(szConfigDir)); + return szConfigDir; + } + if (szConfigDir[0] == '\0') { + hr = p_SHGetFolderPath(NULL, CSIDL_COMMON_APPDATA, NULL, 0, szConfigDir); + if (SUCCEEDED(hr)) { + bstrncat(szConfigDir, "\\Bacula", sizeof(szConfigDir)); + } else { + bstrncpy(szConfigDir, DEFAULT_CONFIGDIR, sizeof(szConfigDir)); + } + } + return szConfigDir; +#endif + + /* * Returns false on error * true on OK, with full_path set to where config file should be */ -static bool +bool find_config_file(const char *config_file, char *full_path, int max_path) { int file_length = strlen(config_file) + 1; @@ -1088,59 +1221,52 @@ find_config_file(const char *config_file, char *full_path, int max_path) * Free configuration resources * */ -void CONFIG::free_resources() +void CONFIG::free_all_resources() { - for (int i=m_r_first; i<=m_r_last; i++) { - free_resource(m_res_head[i-m_r_first], i); - m_res_head[i-m_r_first] = NULL; + RES *next, *res; + if (m_res_head == NULL) { + return; } -} - -RES **CONFIG::save_resources() -{ - int num = m_r_last - m_r_first + 1; - RES **res = (RES **)malloc(num*sizeof(RES *)); - for (int i=0; ifirst; + Dmsg2(500, "i=%d, next=%p\n", i, next); + /* Walk down resource chain freeing them */ + for ( ; next; ) { + res = next; + next = res->res_next; + free_resource(res, i); + } + free(m_res_head[i-m_r_first]->res_list); + free(m_res_head[i-m_r_first]); + m_res_head[i-m_r_first] = NULL; + } } - return res; } -RES **CONFIG::new_res_head() +CONFIG::CONFIG() +: m_cf(NULL), + m_scan_error(NULL), + m_err_type(0), + m_res_all(NULL), + m_res_all_size(0), + m_encode_pass(true), + m_r_first(0), + m_r_last(0), + m_resources(NULL), + m_res_head(NULL) { - int size = (m_r_last - m_r_first + 1) * sizeof(RES *); - RES **res = (RES **)malloc(size); - memset(res, 0, size); - return res; + m_errmsg = get_pool_memory(PM_EMSG); + *m_errmsg = 0; } - -#ifdef xxx -void free_config_resources() -{ - for (int i=r_first; i<=r_last; i++) { - free_resource(res_head[i-r_first], i); - res_head[i-r_first] = NULL; - } +CONFIG::~CONFIG() { + free_all_resources(); + free_pool_memory(m_errmsg); } -RES **save_config_resources() +void CONFIG::encode_password(bool a) { - int num = r_last - r_first + 1; - RES **res = (RES **)malloc(num*sizeof(RES *)); - for (int i=0; ibuf = bstrdup(buf); // printf("buf=%p %s\n", jcr, jcr->buf); jcr1 = (MYJCR *)jcr_chain->insert((void *)jcr, my_compare); @@ -429,7 +429,7 @@ int main() printf("num_items=%d\n", jcr_chain->size()); jcr = (MYJCR *)malloc(sizeof(MYJCR)); - memset(jcr, 0, sizeof(MYJCR)); + bmemzero(jcr, sizeof(MYJCR)); jcr->buf = bstrdup("a"); if ((jcr1=(MYJCR *)jcr_chain->search((void *)jcr, my_compare))) { diff --git a/bacula/src/lib/res.c b/bacula/src/lib/res.c index 74598dc647..414b1c9cbf 100644 --- a/bacula/src/lib/res.c +++ b/bacula/src/lib/res.c @@ -33,7 +33,7 @@ extern int32_t r_first; extern int32_t r_last; extern RES_TABLE resources[]; -extern RES **res_head; +extern RES_HEAD **res_head; brwlock_t res_lock; /* resource lock */ static int res_locked = 0; /* resource chain lock count -- for debug */ @@ -73,23 +73,30 @@ void b_UnlockRes(const char *file, int line) #endif } +/* + * Compare two resource names + */ +int res_compare(void *item1, void *item2) +{ + RES *res1 = (RES *)item1; + RES *res2 = (RES *)item2; + return strcmp(res1->name, res2->name); +} + /* * Return resource of type rcode that matches name */ RES * GetResWithName(int rcode, const char *name) { - RES *res; + RES_HEAD *reshead; int rindex = rcode - r_first; + RES item, *res; LockRes(); - res = res_head[rindex]; - while (res) { - if (strcmp(res->name, name) == 0) { - break; - } - res = res->next; - } + reshead = res_head[rindex]; + item.name = (char *)name; + res = (RES *)reshead->res_list->search(&item, res_compare); UnlockRes(); return res; @@ -107,9 +114,28 @@ GetNextRes(int rcode, RES *res) int rindex = rcode - r_first; if (res == NULL) { - nres = res_head[rindex]; + nres = (RES *)res_head[rindex]->first; + } else { + nres = res->res_next; + } + return nres; +} + +/* + * Return next resource of type rcode. On first + * call second arg (res) is NULL, on subsequent + * calls, it is called with previous value. + */ +RES * +GetNextRes(RES_HEAD **rhead, int rcode, RES *res) +{ + RES *nres; + int rindex = rcode - r_first; + + if (res == NULL) { + nres = (RES *)rhead[rindex]->first; } else { - nres = res->next; + nres = res->res_next; } return nres; } diff --git a/bacula/src/lib/scan.c b/bacula/src/lib/scan.c index c29e82ba03..3eafcd9bf5 100644 --- a/bacula/src/lib/scan.c +++ b/bacula/src/lib/scan.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -35,14 +35,15 @@ void strip_leading_space(char *str) while (B_ISSPACE(*p)) { p++; } - if (p != str) { - strcpy(str, p); + if (str != p) { + do { + *str++ = *p; + } while (*p++ != 0); } } - /* Strip any trailing junk from the command */ -void strip_trailing_junk(char *cmd) +char *strip_trailing_junk(char *cmd) { char *p; @@ -51,24 +52,27 @@ void strip_trailing_junk(char *cmd) while ((p >= cmd) && (B_ISSPACE(*p) || *p == '\n' || *p == '\r')) { *p-- = 0; } + return cmd; } /* Strip any trailing newline characters from the string */ -void strip_trailing_newline(char *cmd) +char *strip_trailing_newline(char *cmd) { char *p; p = cmd - 1 + strlen(cmd); while ((p >= cmd) && (*p == '\n' || *p == '\r')) *p-- = 0; + return cmd; } /* Strip any trailing slashes from a directory path */ -void strip_trailing_slashes(char *dir) +char *strip_trailing_slashes(char *dir) { char *p; /* strip trailing slashes */ p = dir -1 + strlen(dir); while (p >= dir && IsPathSeparator(*p)) *p-- = 0; + return dir; } /* diff --git a/bacula/src/lib/sha2.c b/bacula/src/lib/sha2.c new file mode 100644 index 0000000000..56fb51801a --- /dev/null +++ b/bacula/src/lib/sha2.c @@ -0,0 +1,950 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if 0 +#define UNROLL_LOOPS /* Enable loops unrolling */ +#endif + +#include "sha2.h" + +#ifndef HAVE_SHA2 + +#define SHFR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) +#define CH(x, y, z) ((x & y) ^ (~x & z)) +#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) + +#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3)) +#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10)) + +#define SHA512_F1(x) (ROTR(x, 28) ^ ROTR(x, 34) ^ ROTR(x, 39)) +#define SHA512_F2(x) (ROTR(x, 14) ^ ROTR(x, 18) ^ ROTR(x, 41)) +#define SHA512_F3(x) (ROTR(x, 1) ^ ROTR(x, 8) ^ SHFR(x, 7)) +#define SHA512_F4(x) (ROTR(x, 19) ^ ROTR(x, 61) ^ SHFR(x, 6)) + +#define UNPACK32(x, str) \ +{ \ + *((str) + 3) = (uint8_t) ((x) ); \ + *((str) + 2) = (uint8_t) ((x) >> 8); \ + *((str) + 1) = (uint8_t) ((x) >> 16); \ + *((str) + 0) = (uint8_t) ((x) >> 24); \ +} + +#define PACK32(str, x) \ +{ \ + *(x) = ((uint32_t) *((str) + 3) ) \ + | ((uint32_t) *((str) + 2) << 8) \ + | ((uint32_t) *((str) + 1) << 16) \ + | ((uint32_t) *((str) + 0) << 24); \ +} + +#define UNPACK64(x, str) \ +{ \ + *((str) + 7) = (uint8_t) ((x) ); \ + *((str) + 6) = (uint8_t) ((x) >> 8); \ + *((str) + 5) = (uint8_t) ((x) >> 16); \ + *((str) + 4) = (uint8_t) ((x) >> 24); \ + *((str) + 3) = (uint8_t) ((x) >> 32); \ + *((str) + 2) = (uint8_t) ((x) >> 40); \ + *((str) + 1) = (uint8_t) ((x) >> 48); \ + *((str) + 0) = (uint8_t) ((x) >> 56); \ +} + +#define PACK64(str, x) \ +{ \ + *(x) = ((uint64_t) *((str) + 7) ) \ + | ((uint64_t) *((str) + 6) << 8) \ + | ((uint64_t) *((str) + 5) << 16) \ + | ((uint64_t) *((str) + 4) << 24) \ + | ((uint64_t) *((str) + 3) << 32) \ + | ((uint64_t) *((str) + 2) << 40) \ + | ((uint64_t) *((str) + 1) << 48) \ + | ((uint64_t) *((str) + 0) << 56); \ +} + +/* Macros used for loops unrolling */ + +#define SHA256_SCR(i) \ +{ \ + w[i] = SHA256_F4(w[i - 2]) + w[i - 7] \ + + SHA256_F3(w[i - 15]) + w[i - 16]; \ +} + +#define SHA512_SCR(i) \ +{ \ + w[i] = SHA512_F4(w[i - 2]) + w[i - 7] \ + + SHA512_F3(w[i - 15]) + w[i - 16]; \ +} + +#define SHA256_EXP(a, b, c, d, e, f, g, h, j) \ +{ \ + t1 = wv[h] + SHA256_F2(wv[e]) + CH(wv[e], wv[f], wv[g]) \ + + sha256_k[j] + w[j]; \ + t2 = SHA256_F1(wv[a]) + MAJ(wv[a], wv[b], wv[c]); \ + wv[d] += t1; \ + wv[h] = t1 + t2; \ +} + +#define SHA512_EXP(a, b, c, d, e, f, g ,h, j) \ +{ \ + t1 = wv[h] + SHA512_F2(wv[e]) + CH(wv[e], wv[f], wv[g]) \ + + sha512_k[j] + w[j]; \ + t2 = SHA512_F1(wv[a]) + MAJ(wv[a], wv[b], wv[c]); \ + wv[d] += t1; \ + wv[h] = t1 + t2; \ +} + +uint32_t sha224_h0[8] = + {0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, + 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4}; + +uint32_t sha256_h0[8] = + {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + +uint64_t sha384_h0[8] = + {0xcbbb9d5dc1059ed8ULL, 0x629a292a367cd507ULL, + 0x9159015a3070dd17ULL, 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, 0x8eb44a8768581511ULL, + 0xdb0c2e0d64f98fa7ULL, 0x47b5481dbefa4fa4ULL}; + +uint64_t sha512_h0[8] = + {0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL}; + +uint32_t sha256_k[64] = + {0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +uint64_t sha512_k[80] = + {0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL}; + +/* SHA-256 functions */ + +void sha256_transf(sha256_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint32_t w[64]; + uint32_t wv[8]; + uint32_t t1, t2; + const unsigned char *sub_block; + int i; + +#ifndef UNROLL_LOOPS + int j; +#endif + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 6); + +#ifndef UNROLL_LOOPS + for (j = 0; j < 16; j++) { + PACK32(&sub_block[j << 2], &w[j]); + } + + for (j = 16; j < 64; j++) { + SHA256_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } +#else + PACK32(&sub_block[ 0], &w[ 0]); PACK32(&sub_block[ 4], &w[ 1]); + PACK32(&sub_block[ 8], &w[ 2]); PACK32(&sub_block[12], &w[ 3]); + PACK32(&sub_block[16], &w[ 4]); PACK32(&sub_block[20], &w[ 5]); + PACK32(&sub_block[24], &w[ 6]); PACK32(&sub_block[28], &w[ 7]); + PACK32(&sub_block[32], &w[ 8]); PACK32(&sub_block[36], &w[ 9]); + PACK32(&sub_block[40], &w[10]); PACK32(&sub_block[44], &w[11]); + PACK32(&sub_block[48], &w[12]); PACK32(&sub_block[52], &w[13]); + PACK32(&sub_block[56], &w[14]); PACK32(&sub_block[60], &w[15]); + + SHA256_SCR(16); SHA256_SCR(17); SHA256_SCR(18); SHA256_SCR(19); + SHA256_SCR(20); SHA256_SCR(21); SHA256_SCR(22); SHA256_SCR(23); + SHA256_SCR(24); SHA256_SCR(25); SHA256_SCR(26); SHA256_SCR(27); + SHA256_SCR(28); SHA256_SCR(29); SHA256_SCR(30); SHA256_SCR(31); + SHA256_SCR(32); SHA256_SCR(33); SHA256_SCR(34); SHA256_SCR(35); + SHA256_SCR(36); SHA256_SCR(37); SHA256_SCR(38); SHA256_SCR(39); + SHA256_SCR(40); SHA256_SCR(41); SHA256_SCR(42); SHA256_SCR(43); + SHA256_SCR(44); SHA256_SCR(45); SHA256_SCR(46); SHA256_SCR(47); + SHA256_SCR(48); SHA256_SCR(49); SHA256_SCR(50); SHA256_SCR(51); + SHA256_SCR(52); SHA256_SCR(53); SHA256_SCR(54); SHA256_SCR(55); + SHA256_SCR(56); SHA256_SCR(57); SHA256_SCR(58); SHA256_SCR(59); + SHA256_SCR(60); SHA256_SCR(61); SHA256_SCR(62); SHA256_SCR(63); + + wv[0] = ctx->h[0]; wv[1] = ctx->h[1]; + wv[2] = ctx->h[2]; wv[3] = ctx->h[3]; + wv[4] = ctx->h[4]; wv[5] = ctx->h[5]; + wv[6] = ctx->h[6]; wv[7] = ctx->h[7]; + + SHA256_EXP(0,1,2,3,4,5,6,7, 0); SHA256_EXP(7,0,1,2,3,4,5,6, 1); + SHA256_EXP(6,7,0,1,2,3,4,5, 2); SHA256_EXP(5,6,7,0,1,2,3,4, 3); + SHA256_EXP(4,5,6,7,0,1,2,3, 4); SHA256_EXP(3,4,5,6,7,0,1,2, 5); + SHA256_EXP(2,3,4,5,6,7,0,1, 6); SHA256_EXP(1,2,3,4,5,6,7,0, 7); + SHA256_EXP(0,1,2,3,4,5,6,7, 8); SHA256_EXP(7,0,1,2,3,4,5,6, 9); + SHA256_EXP(6,7,0,1,2,3,4,5,10); SHA256_EXP(5,6,7,0,1,2,3,4,11); + SHA256_EXP(4,5,6,7,0,1,2,3,12); SHA256_EXP(3,4,5,6,7,0,1,2,13); + SHA256_EXP(2,3,4,5,6,7,0,1,14); SHA256_EXP(1,2,3,4,5,6,7,0,15); + SHA256_EXP(0,1,2,3,4,5,6,7,16); SHA256_EXP(7,0,1,2,3,4,5,6,17); + SHA256_EXP(6,7,0,1,2,3,4,5,18); SHA256_EXP(5,6,7,0,1,2,3,4,19); + SHA256_EXP(4,5,6,7,0,1,2,3,20); SHA256_EXP(3,4,5,6,7,0,1,2,21); + SHA256_EXP(2,3,4,5,6,7,0,1,22); SHA256_EXP(1,2,3,4,5,6,7,0,23); + SHA256_EXP(0,1,2,3,4,5,6,7,24); SHA256_EXP(7,0,1,2,3,4,5,6,25); + SHA256_EXP(6,7,0,1,2,3,4,5,26); SHA256_EXP(5,6,7,0,1,2,3,4,27); + SHA256_EXP(4,5,6,7,0,1,2,3,28); SHA256_EXP(3,4,5,6,7,0,1,2,29); + SHA256_EXP(2,3,4,5,6,7,0,1,30); SHA256_EXP(1,2,3,4,5,6,7,0,31); + SHA256_EXP(0,1,2,3,4,5,6,7,32); SHA256_EXP(7,0,1,2,3,4,5,6,33); + SHA256_EXP(6,7,0,1,2,3,4,5,34); SHA256_EXP(5,6,7,0,1,2,3,4,35); + SHA256_EXP(4,5,6,7,0,1,2,3,36); SHA256_EXP(3,4,5,6,7,0,1,2,37); + SHA256_EXP(2,3,4,5,6,7,0,1,38); SHA256_EXP(1,2,3,4,5,6,7,0,39); + SHA256_EXP(0,1,2,3,4,5,6,7,40); SHA256_EXP(7,0,1,2,3,4,5,6,41); + SHA256_EXP(6,7,0,1,2,3,4,5,42); SHA256_EXP(5,6,7,0,1,2,3,4,43); + SHA256_EXP(4,5,6,7,0,1,2,3,44); SHA256_EXP(3,4,5,6,7,0,1,2,45); + SHA256_EXP(2,3,4,5,6,7,0,1,46); SHA256_EXP(1,2,3,4,5,6,7,0,47); + SHA256_EXP(0,1,2,3,4,5,6,7,48); SHA256_EXP(7,0,1,2,3,4,5,6,49); + SHA256_EXP(6,7,0,1,2,3,4,5,50); SHA256_EXP(5,6,7,0,1,2,3,4,51); + SHA256_EXP(4,5,6,7,0,1,2,3,52); SHA256_EXP(3,4,5,6,7,0,1,2,53); + SHA256_EXP(2,3,4,5,6,7,0,1,54); SHA256_EXP(1,2,3,4,5,6,7,0,55); + SHA256_EXP(0,1,2,3,4,5,6,7,56); SHA256_EXP(7,0,1,2,3,4,5,6,57); + SHA256_EXP(6,7,0,1,2,3,4,5,58); SHA256_EXP(5,6,7,0,1,2,3,4,59); + SHA256_EXP(4,5,6,7,0,1,2,3,60); SHA256_EXP(3,4,5,6,7,0,1,2,61); + SHA256_EXP(2,3,4,5,6,7,0,1,62); SHA256_EXP(1,2,3,4,5,6,7,0,63); + + ctx->h[0] += wv[0]; ctx->h[1] += wv[1]; + ctx->h[2] += wv[2]; ctx->h[3] += wv[3]; + ctx->h[4] += wv[4]; ctx->h[5] += wv[5]; + ctx->h[6] += wv[6]; ctx->h[7] += wv[7]; +#endif /* !UNROLL_LOOPS */ + } +} + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest) +{ + sha256_ctx ctx; + + sha256_init(&ctx); + sha256_update(&ctx, message, len); + sha256_final(&ctx, digest); +} + +void sha256_init(sha256_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha256_h0[i]; + } +#else + ctx->h[0] = sha256_h0[0]; ctx->h[1] = sha256_h0[1]; + ctx->h[2] = sha256_h0[2]; ctx->h[3] = sha256_h0[3]; + ctx->h[4] = sha256_h0[4]; ctx->h[5] = sha256_h0[5]; + ctx->h[6] = sha256_h0[6]; ctx->h[7] = sha256_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA256_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA256_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA256_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA256_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha256_final(sha256_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) + < (ctx->len % SHA256_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 8; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +#else + UNPACK32(ctx->h[0], &digest[ 0]); + UNPACK32(ctx->h[1], &digest[ 4]); + UNPACK32(ctx->h[2], &digest[ 8]); + UNPACK32(ctx->h[3], &digest[12]); + UNPACK32(ctx->h[4], &digest[16]); + UNPACK32(ctx->h[5], &digest[20]); + UNPACK32(ctx->h[6], &digest[24]); + UNPACK32(ctx->h[7], &digest[28]); +#endif /* !UNROLL_LOOPS */ +} + +/* SHA-512 functions */ + +void sha512_transf(sha512_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint64_t w[80]; + uint64_t wv[8]; + uint64_t t1, t2; + const unsigned char *sub_block; + int i, j; + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 7); + +#ifndef UNROLL_LOOPS + for (j = 0; j < 16; j++) { + PACK64(&sub_block[j << 3], &w[j]); + } + + for (j = 16; j < 80; j++) { + SHA512_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 80; j++) { + t1 = wv[7] + SHA512_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha512_k[j] + w[j]; + t2 = SHA512_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } +#else + PACK64(&sub_block[ 0], &w[ 0]); PACK64(&sub_block[ 8], &w[ 1]); + PACK64(&sub_block[ 16], &w[ 2]); PACK64(&sub_block[ 24], &w[ 3]); + PACK64(&sub_block[ 32], &w[ 4]); PACK64(&sub_block[ 40], &w[ 5]); + PACK64(&sub_block[ 48], &w[ 6]); PACK64(&sub_block[ 56], &w[ 7]); + PACK64(&sub_block[ 64], &w[ 8]); PACK64(&sub_block[ 72], &w[ 9]); + PACK64(&sub_block[ 80], &w[10]); PACK64(&sub_block[ 88], &w[11]); + PACK64(&sub_block[ 96], &w[12]); PACK64(&sub_block[104], &w[13]); + PACK64(&sub_block[112], &w[14]); PACK64(&sub_block[120], &w[15]); + + SHA512_SCR(16); SHA512_SCR(17); SHA512_SCR(18); SHA512_SCR(19); + SHA512_SCR(20); SHA512_SCR(21); SHA512_SCR(22); SHA512_SCR(23); + SHA512_SCR(24); SHA512_SCR(25); SHA512_SCR(26); SHA512_SCR(27); + SHA512_SCR(28); SHA512_SCR(29); SHA512_SCR(30); SHA512_SCR(31); + SHA512_SCR(32); SHA512_SCR(33); SHA512_SCR(34); SHA512_SCR(35); + SHA512_SCR(36); SHA512_SCR(37); SHA512_SCR(38); SHA512_SCR(39); + SHA512_SCR(40); SHA512_SCR(41); SHA512_SCR(42); SHA512_SCR(43); + SHA512_SCR(44); SHA512_SCR(45); SHA512_SCR(46); SHA512_SCR(47); + SHA512_SCR(48); SHA512_SCR(49); SHA512_SCR(50); SHA512_SCR(51); + SHA512_SCR(52); SHA512_SCR(53); SHA512_SCR(54); SHA512_SCR(55); + SHA512_SCR(56); SHA512_SCR(57); SHA512_SCR(58); SHA512_SCR(59); + SHA512_SCR(60); SHA512_SCR(61); SHA512_SCR(62); SHA512_SCR(63); + SHA512_SCR(64); SHA512_SCR(65); SHA512_SCR(66); SHA512_SCR(67); + SHA512_SCR(68); SHA512_SCR(69); SHA512_SCR(70); SHA512_SCR(71); + SHA512_SCR(72); SHA512_SCR(73); SHA512_SCR(74); SHA512_SCR(75); + SHA512_SCR(76); SHA512_SCR(77); SHA512_SCR(78); SHA512_SCR(79); + + wv[0] = ctx->h[0]; wv[1] = ctx->h[1]; + wv[2] = ctx->h[2]; wv[3] = ctx->h[3]; + wv[4] = ctx->h[4]; wv[5] = ctx->h[5]; + wv[6] = ctx->h[6]; wv[7] = ctx->h[7]; + + j = 0; + + do { + SHA512_EXP(0,1,2,3,4,5,6,7,j); j++; + SHA512_EXP(7,0,1,2,3,4,5,6,j); j++; + SHA512_EXP(6,7,0,1,2,3,4,5,j); j++; + SHA512_EXP(5,6,7,0,1,2,3,4,j); j++; + SHA512_EXP(4,5,6,7,0,1,2,3,j); j++; + SHA512_EXP(3,4,5,6,7,0,1,2,j); j++; + SHA512_EXP(2,3,4,5,6,7,0,1,j); j++; + SHA512_EXP(1,2,3,4,5,6,7,0,j); j++; + } while (j < 80); + + ctx->h[0] += wv[0]; ctx->h[1] += wv[1]; + ctx->h[2] += wv[2]; ctx->h[3] += wv[3]; + ctx->h[4] += wv[4]; ctx->h[5] += wv[5]; + ctx->h[6] += wv[6]; ctx->h[7] += wv[7]; +#endif /* !UNROLL_LOOPS */ + } +} + +void sha512(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha512_ctx ctx; + + sha512_init(&ctx); + sha512_update(&ctx, message, len); + sha512_final(&ctx, digest); +} + +void sha512_init(sha512_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha512_h0[i]; + } +#else + ctx->h[0] = sha512_h0[0]; ctx->h[1] = sha512_h0[1]; + ctx->h[2] = sha512_h0[2]; ctx->h[3] = sha512_h0[3]; + ctx->h[4] = sha512_h0[4]; ctx->h[5] = sha512_h0[5]; + ctx->h[6] = sha512_h0[6]; ctx->h[7] = sha512_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha512_update(sha512_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA512_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA512_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA512_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha512_transf(ctx, ctx->block, 1); + sha512_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA512_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 7], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 7; +} + +void sha512_final(sha512_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = 1 + ((SHA512_BLOCK_SIZE - 17) + < (ctx->len % SHA512_BLOCK_SIZE)); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 7; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha512_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 8; i++) { + UNPACK64(ctx->h[i], &digest[i << 3]); + } +#else + UNPACK64(ctx->h[0], &digest[ 0]); + UNPACK64(ctx->h[1], &digest[ 8]); + UNPACK64(ctx->h[2], &digest[16]); + UNPACK64(ctx->h[3], &digest[24]); + UNPACK64(ctx->h[4], &digest[32]); + UNPACK64(ctx->h[5], &digest[40]); + UNPACK64(ctx->h[6], &digest[48]); + UNPACK64(ctx->h[7], &digest[56]); +#endif /* !UNROLL_LOOPS */ +} + +/* SHA-384 functions */ + +void sha384(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha384_ctx ctx; + + sha384_init(&ctx); + sha384_update(&ctx, message, len); + sha384_final(&ctx, digest); +} + +void sha384_init(sha384_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha384_h0[i]; + } +#else + ctx->h[0] = sha384_h0[0]; ctx->h[1] = sha384_h0[1]; + ctx->h[2] = sha384_h0[2]; ctx->h[3] = sha384_h0[3]; + ctx->h[4] = sha384_h0[4]; ctx->h[5] = sha384_h0[5]; + ctx->h[6] = sha384_h0[6]; ctx->h[7] = sha384_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha384_update(sha384_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA384_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA384_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA384_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha512_transf(ctx, ctx->block, 1); + sha512_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA384_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 7], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 7; +} + +void sha384_final(sha384_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = (1 + ((SHA384_BLOCK_SIZE - 17) + < (ctx->len % SHA384_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 7; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha512_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 6; i++) { + UNPACK64(ctx->h[i], &digest[i << 3]); + } +#else + UNPACK64(ctx->h[0], &digest[ 0]); + UNPACK64(ctx->h[1], &digest[ 8]); + UNPACK64(ctx->h[2], &digest[16]); + UNPACK64(ctx->h[3], &digest[24]); + UNPACK64(ctx->h[4], &digest[32]); + UNPACK64(ctx->h[5], &digest[40]); +#endif /* !UNROLL_LOOPS */ +} + +/* SHA-224 functions */ + +void sha224(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha224_ctx ctx; + + sha224_init(&ctx); + sha224_update(&ctx, message, len); + sha224_final(&ctx, digest); +} + +void sha224_init(sha224_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha224_h0[i]; + } +#else + ctx->h[0] = sha224_h0[0]; ctx->h[1] = sha224_h0[1]; + ctx->h[2] = sha224_h0[2]; ctx->h[3] = sha224_h0[3]; + ctx->h[4] = sha224_h0[4]; ctx->h[5] = sha224_h0[5]; + ctx->h[6] = sha224_h0[6]; ctx->h[7] = sha224_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha224_update(sha224_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA224_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA224_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA224_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA224_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha224_final(sha224_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = (1 + ((SHA224_BLOCK_SIZE - 9) + < (ctx->len % SHA224_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 7; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +#else + UNPACK32(ctx->h[0], &digest[ 0]); + UNPACK32(ctx->h[1], &digest[ 4]); + UNPACK32(ctx->h[2], &digest[ 8]); + UNPACK32(ctx->h[3], &digest[12]); + UNPACK32(ctx->h[4], &digest[16]); + UNPACK32(ctx->h[5], &digest[20]); + UNPACK32(ctx->h[6], &digest[24]); +#endif /* !UNROLL_LOOPS */ +} +#endif /* HAVE_SHA2 */ + +#ifdef TEST_VECTORS + +/* FIPS 180-2 Validation tests */ + +#include +#include + +void test(const char *vector, unsigned char *digest, + unsigned int digest_size) +{ + char output[2 * SHA512_DIGEST_SIZE + 1]; + int i; + + output[2 * digest_size] = '\0'; + + for (i = 0; i < (int) digest_size ; i++) { + sprintf(output + 2 * i, "%02x", digest[i]); + } + + printf("H: %s\n", output); + if (strcmp(vector, output)) { + fprintf(stderr, "Test failed.\n"); + exit(EXIT_FAILURE); + } +} + +int main(void) +{ + static const char *vectors[4][3] = + { /* SHA-224 */ + { + "23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", + "75388b16512776cc5dba5da1fd890150b0c6455cb4f58b1952522525", + "20794655980c91d8bbb4c1ea97618a4bf03f42581948b2ee4ee7ad67", + }, + /* SHA-256 */ + { + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0", + }, + /* SHA-384 */ + { + "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed" + "8086072ba1e7cc2358baeca134c825a7", + "09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712" + "fcc7c71a557e2db966c3e9fa91746039", + "9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b" + "07b8b3dc38ecc4ebae97ddd87f3d8985", + }, + /* SHA-512 */ + { + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a" + "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", + "8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018" + "501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909", + "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb" + "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b" + } + }; + + static const char message1[] = "abc"; + static const char message2a[] = "abcdbcdecdefdefgefghfghighijhi" + "jkijkljklmklmnlmnomnopnopq"; + static const char message2b[] = "abcdefghbcdefghicdefghijdefghijkefghij" + "klfghijklmghijklmnhijklmnoijklmnopjklm" + "nopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + unsigned char *message3; + unsigned int message3_len = 1000000; + unsigned char digest[SHA512_DIGEST_SIZE]; + + message3 = malloc(message3_len); + if (message3 == NULL) { + fprintf(stderr, "Can't allocate memory\n"); + return -1; + } + memset(message3, 'a', message3_len); + + printf("SHA-2 FIPS 180-2 Validation tests\n\n"); + printf("SHA-224 Test vectors\n"); + + sha224((const unsigned char *) message1, strlen(message1), digest); + test(vectors[0][0], digest, SHA224_DIGEST_SIZE); + sha224((const unsigned char *) message2a, strlen(message2a), digest); + test(vectors[0][1], digest, SHA224_DIGEST_SIZE); + sha224(message3, message3_len, digest); + test(vectors[0][2], digest, SHA224_DIGEST_SIZE); + printf("\n"); + + printf("SHA-256 Test vectors\n"); + + sha256((const unsigned char *) message1, strlen(message1), digest); + test(vectors[1][0], digest, SHA256_DIGEST_SIZE); + sha256((const unsigned char *) message2a, strlen(message2a), digest); + test(vectors[1][1], digest, SHA256_DIGEST_SIZE); + sha256(message3, message3_len, digest); + test(vectors[1][2], digest, SHA256_DIGEST_SIZE); + printf("\n"); + + printf("SHA-384 Test vectors\n"); + + sha384((const unsigned char *) message1, strlen(message1), digest); + test(vectors[2][0], digest, SHA384_DIGEST_SIZE); + sha384((const unsigned char *)message2b, strlen(message2b), digest); + test(vectors[2][1], digest, SHA384_DIGEST_SIZE); + sha384(message3, message3_len, digest); + test(vectors[2][2], digest, SHA384_DIGEST_SIZE); + printf("\n"); + + printf("SHA-512 Test vectors\n"); + + sha512((const unsigned char *) message1, strlen(message1), digest); + test(vectors[3][0], digest, SHA512_DIGEST_SIZE); + sha512((const unsigned char *) message2b, strlen(message2b), digest); + test(vectors[3][1], digest, SHA512_DIGEST_SIZE); + sha512(message3, message3_len, digest); + test(vectors[3][2], digest, SHA512_DIGEST_SIZE); + printf("\n"); + + printf("All tests passed.\n"); + + return 0; +} + +#endif /* TEST_VECTORS */ + diff --git a/bacula/src/lib/sha2.h b/bacula/src/lib/sha2.h new file mode 100644 index 0000000000..8c29f5bedd --- /dev/null +++ b/bacula/src/lib/sha2.h @@ -0,0 +1,118 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef SHA2_H +#define SHA2_H + +#include "bacula.h" + +#ifndef HAVE_SHA2 + +#define SHA224_DIGEST_SIZE ( 224 / 8) +#define SHA256_DIGEST_SIZE ( 256 / 8) +#define SHA384_DIGEST_SIZE ( 384 / 8) +#define SHA512_DIGEST_SIZE ( 512 / 8) + +#define SHA256_BLOCK_SIZE ( 512 / 8) +#define SHA512_BLOCK_SIZE (1024 / 8) +#define SHA384_BLOCK_SIZE SHA512_BLOCK_SIZE +#define SHA224_BLOCK_SIZE SHA256_BLOCK_SIZE + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA256_BLOCK_SIZE]; + uint32_t h[8]; +} sha256_ctx; + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA512_BLOCK_SIZE]; + uint64_t h[8]; +} sha512_ctx; + +typedef sha512_ctx sha384_ctx; +typedef sha256_ctx sha224_ctx; + +void sha224_init(sha224_ctx *ctx); +void sha224_update(sha224_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha224_final(sha224_ctx *ctx, unsigned char *digest); +void sha224(const unsigned char *message, unsigned int len, + unsigned char *digest); + +void sha256_init(sha256_ctx * ctx); +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha256_final(sha256_ctx *ctx, unsigned char *digest); +void sha256(const unsigned char *message, unsigned int len, + unsigned char *digest); + +void sha384_init(sha384_ctx *ctx); +void sha384_update(sha384_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha384_final(sha384_ctx *ctx, unsigned char *digest); +void sha384(const unsigned char *message, unsigned int len, + unsigned char *digest); + +void sha512_init(sha512_ctx *ctx); +void sha512_update(sha512_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha512_final(sha512_ctx *ctx, unsigned char *digest); +void sha512(const unsigned char *message, unsigned int len, + unsigned char *digest); + +#ifdef __cplusplus +} +#endif + +/* implement the openssl fall back */ +typedef sha256_ctx SHA256_CTX; +#define SHA256_Init(ctx) sha256_init(ctx) +#define SHA256_Update(ctx, msg, len) sha256_update(ctx, (const unsigned char *)(msg), len) +#define SHA256_Final(digest, ctx) sha256_final(ctx, digest) + +typedef sha512_ctx SHA512_CTX; +#define SHA512_Init(ctx) sha512_init(ctx) +#define SHA512_Update(ctx, msg, len) sha512_update(ctx, (const unsigned char *)(msg), len) +#define SHA512_Final(digest, ctx) sha512_final(ctx, digest) + +#endif /* HAVE_SHA2 */ + +#endif /* !SHA2_H */ + diff --git a/bacula/src/lib/signal.c b/bacula/src/lib/signal.c index 4125839798..0ca3af9903 100644 --- a/bacula/src/lib/signal.c +++ b/bacula/src/lib/signal.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -69,6 +69,16 @@ extern void dbg_print_plugin(FILE *fp); /* defined in lockmgr.c */ extern void dbg_print_lock(FILE *fp); +#define MAX_DBG_HOOK 10 +static dbg_hook_t *dbg_hooks[MAX_DBG_HOOK]; +static int dbg_handler_count=0; + +void dbg_add_hook(dbg_hook_t *hook) +{ + ASSERT(dbg_handler_count < MAX_DBG_HOOK); + dbg_hooks[dbg_handler_count++] = hook; +} + /* * !!! WARNING !!! * @@ -79,14 +89,13 @@ static void dbg_print_bacula() { char buf[512]; - snprintf(buf, sizeof(buf), "%s/%s.%d.lockdump", - working_directory, my_name, (int)getpid()); + snprintf(buf, sizeof(buf), "%s/bacula.%d.traceback", working_directory, main_pid); FILE *fp = fopen(buf, "a+") ; if (!fp) { fp = stderr; } - fprintf(stderr, "Dumping: %s\n", buf); + fprintf(stderr, "LockDump: %s\n", buf); /* Print also BDB and RWLOCK structure * Can add more info about JCR with dbg_jcr_add_hook() @@ -95,26 +104,11 @@ static void dbg_print_bacula() dbg_print_jcr(fp); dbg_print_plugin(fp); + for(int i=0; i < dbg_handler_count ; i++) { + dbg_hooks[i](fp); + } + if (fp != stderr) { -#define direct_print -#ifdef direct_print - if (prt_kaboom) { - rewind(fp); - printf("\n\n ==== lockdump output ====\n\n"); - while (fgets(buf, (int)sizeof(buf), fp) != NULL) { - printf("%s", buf); - } - printf(" ==== End baktrace output ====\n\n"); - } -#else - if (prt_kaboom) { - char buf1[512]; - printf("\n\n ==== lockdump output ====\n\n"); - snprintf(buf1, sizeof(buf1), "/bin/cat %s", buf); - system(buf1); - printf(" ==== End baktrace output ====\n\n"); - } -#endif fclose(fp); } } @@ -128,10 +122,6 @@ extern "C" void signal_handler(int sig) int chld_status=-1; utime_t now; - /* If we come back more than once, get out fast! */ - if (already_dead) { - exit(1); - } Dmsg2(900, "sig=%d %s\n", sig, sig_names[sig]); /* Ignore certain signals -- SIGUSR2 used to interrupt threads */ if (sig == SIGCHLD || sig == SIGUSR2) { @@ -141,6 +131,10 @@ extern "C" void signal_handler(int sig) if (sig == 0) { return; } + /* If we come back more than once, get out fast! */ + if (already_dead) { + exit(1); + } already_dead++; /* Don't use Emsg here as it may lock and thus block us */ if (sig == SIGTERM || sig == SIGINT) { @@ -195,13 +189,15 @@ extern "C" void signal_handler(int sig) } unlink("./core"); /* get rid of any old core file */ + sprintf(pid_buf, "%d", (int)main_pid); + snprintf(buf, sizeof(buf), "%s/bacula.%s.traceback", working_directory, pid_buf); + unlink(buf); /* Remove the previous backtrace file if exist */ + #ifdef DEVELOPER /* When DEVELOPER not set, this is done below */ - /* print information about the current state into working/.lockdump */ + /* print information about the current state into working/.traceback */ dbg_print_bacula(); #endif - - sprintf(pid_buf, "%d", (int)main_pid); Dmsg1(300, "Working=%s\n", working_directory); Dmsg1(300, "btpath=%s\n", btpath); Dmsg1(300, "exepath=%s\n", exepath); @@ -246,11 +242,17 @@ extern "C" void signal_handler(int sig) fprintf(stderr, _("The btraceback call returned %d\n"), WEXITSTATUS(chld_status)); } + +#ifndef DEVELOPER /* When DEVELOPER set, this is done above */ + /* print information about the current state into working/.traceback */ + dbg_print_bacula(); +#endif + /* If we want it printed, do so */ #ifdef direct_print if (prt_kaboom) { FILE *fd; - snprintf(buf, sizeof(buf), "%s/%s.%s.traceback", working_directory, my_name, pid_buf); + snprintf(buf, sizeof(buf), "%s/bacula.%s.traceback", working_directory, pid_buf); fd = fopen(buf, "r"); if (fd != NULL) { printf("\n\n ==== Traceback output ====\n\n"); @@ -263,18 +265,13 @@ extern "C" void signal_handler(int sig) } #else if (prt_kaboom) { - snprintf(buf, sizeof(buf), "/bin/cat %s/%s.%s.traceback", working_directory, my_name, pid_buf); + snprintf(buf, sizeof(buf), "/bin/cat %s/bacula.%s.traceback", working_directory, pid_buf); fprintf(stderr, "\n\n ==== Traceback output ====\n\n"); system(buf); fprintf(stderr, " ==== End traceback output ====\n\n"); } #endif -#ifndef DEVELOPER /* When DEVELOPER set, this is done above */ - /* print information about the current state into working/.lockdump */ - dbg_print_bacula(); -#endif - } #endif exit_handler(sig); diff --git a/bacula/src/lib/smartall.c b/bacula/src/lib/smartall.c index 0d4ecb73ac..61282d68a6 100644 --- a/bacula/src/lib/smartall.c +++ b/bacula/src/lib/smartall.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -109,7 +109,10 @@ static void *smalloc(const char *fname, int lineno, unsigned int nbytes) is desired than to miss all the erroneous occurrences where buffer length calculation code results in a zero. */ - ASSERT(nbytes > 0); + if (nbytes == 0) { + Tmsg3(0, "Invalid memory allocation. %u bytes %s:%d\n", nbytes, fname, lineno); + ASSERT(nbytes > 0); + } nbytes += HEAD_SIZE + 1; if ((buf = (char *)malloc(nbytes)) != NULL) { diff --git a/bacula/src/lib/smartall.h b/bacula/src/lib/smartall.h index 4c6921174c..2bde7620e8 100644 --- a/bacula/src/lib/smartall.h +++ b/bacula/src/lib/smartall.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -61,8 +61,8 @@ extern int sm_check_rtn(const char *fname, int lineno, bool bufdump); /* Redefine standard memory allocator calls to use our routines instead. */ -#define free(x) sm_free(__FILE__, __LINE__, (x)) -#define cfree(x) sm_free(__FILE__, __LINE__, (x)) +#define free(x) sm_free(__FILE__, __LINE__, (void *)(x)) +#define cfree(x) sm_free(__FILE__, __LINE__, (void *)(x)) #define malloc(x) sm_malloc(__FILE__, __LINE__, (x)) #define calloc(n,e) sm_calloc(__FILE__, __LINE__, (n), (e)) #define realloc(p,x) sm_realloc(__FILE__, __LINE__, (p), (x)) diff --git a/bacula/src/lib/status.h b/bacula/src/lib/status.h index d38bf4334e..1af792b0cf 100644 --- a/bacula/src/lib/status.h +++ b/bacula/src/lib/status.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -69,10 +69,12 @@ static void sendit(const char *msg, int len, STATUS_PKT *sp) /* common to SD/FD */ static void list_terminated_jobs(STATUS_PKT *sp) { + OutputWriter ow(sp->api_opts); char dt[MAX_TIME_LENGTH], b1[30], b2[30]; char level[10]; struct s_last_job *je; const char *msg; + char *p; msg = _("\nTerminated Jobs:\n"); if (!sp->api) sendit(msg, strlen(msg), sp); @@ -85,6 +87,10 @@ static void list_terminated_jobs(STATUS_PKT *sp) if (!sp->api) sendit(msg, strlen(msg), sp); msg = _("===================================================================\n"); if (!sp->api) sendit(msg, strlen(msg), sp); + if (sp->api > 1) { + p = ow.start_group("terminated"); + sendit(p, strlen(p), sp); + } foreach_dlist(je, last_jobs) { char JobName[MAX_NAME_LENGTH]; const char *termstat; @@ -93,9 +99,10 @@ static void list_terminated_jobs(STATUS_PKT *sp) bstrftime_nc(dt, sizeof(dt), je->end_time); switch (je->JobType) { case JT_ADMIN: - bstrncpy(level, "Admin", sizeof(level)); + bstrncpy(level, "Admn", sizeof(level)); + break; case JT_RESTORE: - bstrncpy(level, "Restore", sizeof(level)); + bstrncpy(level, "Rest", sizeof(level)); break; default: bstrncpy(level, job_level_to_str(je->JobLevel), sizeof(level)); @@ -145,6 +152,27 @@ static void list_terminated_jobs(STATUS_PKT *sp) edit_uint64_with_suffix(je->JobBytes, b2), termstat, dt, JobName); + + } else if (sp->api > 1) { + p = ow.get_output(OT_CLEAR, + OT_START_OBJ, + OT_INT, "jobid", je->JobId, + OT_JOBLEVEL,"level", je->JobLevel, + OT_JOBTYPE, "type", je->JobType, + OT_JOBSTATUS,"status", je->JobStatus, + OT_STRING, "status_desc",termstat, + OT_SIZE, "jobbytes", je->JobBytes, + OT_INT32, "jobfiles", je->JobFiles, + OT_STRING, "job", je->Job, + OT_STRING, "name", JobName, + OT_UTIME, "starttime", je->start_time, + OT_UTIME, "endtime", je->end_time, + OT_INT, "errors", je->Errors, + OT_END_OBJ, + OT_END); + sendit(p, strlen(p), sp); + + } else { bsnprintf(buf, sizeof(buf), _("%6d %-7s %8s %10s %-7s %-8s %s\n"), je->JobId, @@ -159,6 +187,9 @@ static void list_terminated_jobs(STATUS_PKT *sp) unlock_last_jobs_list(); if (!sp->api) { sendit("====\n", 5, sp); + } else if (sp->api > 1) { + p = ow.end_group(false); + sendit(p, strlen(p), sp); } } diff --git a/bacula/src/lib/tls.c b/bacula/src/lib/tls.c index 2fb8ec1edd..f444a276a5 100644 --- a/bacula/src/lib/tls.c +++ b/bacula/src/lib/tls.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -496,14 +496,8 @@ static inline bool openssl_bsock_session_start(BSOCK *bsock, bool server) { TLS_CONNECTION *tls = bsock->tls; int err; - int fdmax, flags; + int flags; int stat = true; - fd_set fdset; - struct timeval tv; - - /* Zero the fdset, we'll set our fd prior to each invocation of select() */ - FD_ZERO(&fdset); - fdmax = bsock->m_fd + 1; /* Ensure that socket is non-blocking */ flags = bsock->set_nonblocking(); @@ -531,22 +525,12 @@ static inline bool openssl_bsock_session_start(BSOCK *bsock, bool server) stat = false; goto cleanup; case SSL_ERROR_WANT_READ: - /* If we timeout of a select, this will be unset */ - FD_SET((unsigned) bsock->m_fd, &fdset); - /* Set our timeout */ - tv.tv_sec = 10; - tv.tv_usec = 0; /* Block until we can read */ - select(fdmax, &fdset, NULL, NULL, &tv); + fd_wait_data(bsock->m_fd, WAIT_READ, 10, 0); break; case SSL_ERROR_WANT_WRITE: - /* If we timeout of a select, this will be unset */ - FD_SET((unsigned) bsock->m_fd, &fdset); - /* Set our timeout */ - tv.tv_sec = 10; - tv.tv_usec = 0; /* Block until we can write */ - select(fdmax, NULL, &fdset, NULL, &tv); + fd_wait_data(bsock->m_fd, WAIT_WRITE, 10, 0); break; default: /* Socket Error Occurred */ @@ -643,16 +627,10 @@ void tls_bsock_shutdown(BSOCK *bsock) static inline int openssl_bsock_readwrite(BSOCK *bsock, char *ptr, int nbytes, bool write) { TLS_CONNECTION *tls = bsock->tls; - int fdmax, flags; - fd_set fdset; - struct timeval tv; + int flags; int nleft = 0; int nwritten = 0; - /* Zero the fdset, we'll set our fd prior to each invocation of select() */ - FD_ZERO(&fdset); - fdmax = bsock->m_fd + 1; - /* Ensure that socket is non-blocking */ flags = bsock->set_nonblocking(); @@ -699,21 +677,13 @@ static inline int openssl_bsock_readwrite(BSOCK *bsock, char *ptr, int nbytes, b goto cleanup; case SSL_ERROR_WANT_READ: - /* If we timeout on a select, this will be unset */ - FD_SET((unsigned)bsock->m_fd, &fdset); - tv.tv_sec = 10; - tv.tv_usec = 0; /* Block until we can read */ - select(fdmax, &fdset, NULL, NULL, &tv); + fd_wait_data(bsock->m_fd, WAIT_READ, 10, 0); break; case SSL_ERROR_WANT_WRITE: - /* If we timeout on a select, this will be unset */ - FD_SET((unsigned)bsock->m_fd, &fdset); - tv.tv_sec = 10; - tv.tv_usec = 0; - /* Block until we can write */ - select(fdmax, NULL, &fdset, NULL, &tv); + /* Block until we can read */ + fd_wait_data(bsock->m_fd, WAIT_WRITE, 10, 0); break; case SSL_ERROR_ZERO_RETURN: diff --git a/bacula/src/lib/tree.c b/bacula/src/lib/tree.c index 04e90c4f4b..99baddbccd 100644 --- a/bacula/src/lib/tree.c +++ b/bacula/src/lib/tree.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -328,6 +328,7 @@ static TREE_NODE *search_and_insert_tree_node(char *fname, int type, strcpy(node->fname, fname); node->parent = parent; node->type = type; + node->can_access = true; /* Maintain a linear chain of nodes */ if (!root->first) { @@ -434,6 +435,9 @@ TREE_NODE *tree_relcwd(char *path, TREE_ROOT *root, TREE_NODE *node) if (!cd || (cd->type == TN_FILE && !tree_node_has_child(cd))) { return NULL; } + if (!cd->can_access) { /* Will display permission denied */ + return cd; + } if (!p) { Dmsg0(100, "tree_relcwd: no more to lookup. found.\n"); return cd; diff --git a/bacula/src/lib/tree.h b/bacula/src/lib/tree.h index 5451eb2ea3..1c6681c261 100644 --- a/bacula/src/lib/tree.h +++ b/bacula/src/lib/tree.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -70,6 +70,7 @@ struct s_tree_node { unsigned int soft_link: 1; /* set if is soft link */ unsigned int inserted: 1; /* set when node newly inserted */ unsigned int loaded: 1; /* set when the dir is in the tree */ + unsigned int can_access: 1; /* Can access to this node */ struct s_tree_node *parent; struct s_tree_node *next; /* next hash of FileIndex */ struct delta_list *delta_list; /* delta parts for this node */ diff --git a/bacula/src/lib/util.c b/bacula/src/lib/util.c index 88de1b0cac..12b3d35420 100644 --- a/bacula/src/lib/util.c +++ b/bacula/src/lib/util.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -31,6 +31,11 @@ * */ +bool is_null(const void *ptr) +{ + return ptr == NULL; +} + /* Return true of buffer has all zero bytes */ bool is_buf_zero(const char *buf, int len) { @@ -60,6 +65,15 @@ bool is_buf_zero(const char *buf, int len) return true; } +/* + * Subroutine that cannot be suppressed by GCC 6.0 + */ +void bmemzero(void *buf, size_t size) +{ + memset(buf, 0, size); + return; +} + /* Convert a string in place to lower case */ void lcase(char *str) @@ -167,12 +181,12 @@ static char hexatable[]="0123456789abcdef"; * ==> * msglen=36 msg=12345678 12345678 */ -char *hexdump(const char *data, int len, char *buf, int capacity) +char *hexdump(const char *data, int len, char *buf, int capacity, bool add_spaces) { char *b=buf; int i=0; while (i2) { - if (i>0 && i%4==0) { + if (add_spaces && i>0 && i%4==0 ) { *(b++)=' '; capacity--; } @@ -253,6 +267,17 @@ char *smartdump(const char *data, int len, char *buf, int capacity, bool *is_asc return buf; } +/* + * check if x is a power two + */ +int is_power_of_two(uint64_t x) +{ + while ( x%2 == 0 && x > 1) { + x /= 2; + } + return (x == 1); +} + /* * Convert a JobStatus code into a human readable form */ @@ -428,6 +453,12 @@ const char *job_status_to_str(int status, int errors) case JS_Differences: str = _("Differences"); break; + case JS_Created: + str = _("Created"); + break; + case JS_Incomplete: + str = _("Incomplete"); + break; default: str = _("Unknown term code"); break; @@ -805,6 +836,15 @@ void decode_session_key(char *decode, char *session, char *key, int maxlen) * %j = Unique Job id * %l = job level * %n = Unadorned Job name + * %p = Pool name (Director) + * %P = Process PID + * %w = Write Store (Director) + * %x = Spool Data (Director) + * %D = Director name (Director/FileDaemon) + * %C = Cloned (Director) + * %I = wjcr->JobId (Director) + * %f = FileSet (Director) + * %h = Client Address (Director) * %s = Since time * %t = Job type (Backup, ...) * %r = Recipients @@ -813,6 +853,7 @@ void decode_session_key(char *decode, char *session, char *key, int maxlen) * %F = Job Files * %E = Job Errors * %R = Job ReadBytes + * %S = Previous Job name (FileDaemon) for Incremental/Differential * * omsg = edited output message * imsg = input string containing edit codes (%x) @@ -935,7 +976,7 @@ POOLMEM *edit_job_codes(JCR *jcr, char *omsg, char *imsg, const char *to, job_co default: str = NULL; if (callback != NULL) { - str = callback(jcr, p); + str = callback(jcr, p, add, sizeof(add)); } if (!str) { diff --git a/bacula/src/lib/watchdog.c b/bacula/src/lib/watchdog.c index 5a7cd9cf4e..4b7afeb3a3 100644 --- a/bacula/src/lib/watchdog.c +++ b/bacula/src/lib/watchdog.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -43,7 +43,7 @@ static void wd_lock(); static void wd_unlock(); /* Static globals */ -static bool quit = false;; +static bool quit = false; static bool wd_is_init = false; static brwlock_t lock; /* watchdog lock */ @@ -256,19 +256,6 @@ extern "C" void *watchdog_thread(void *arg) while (!quit) { watchdog_t *p; - /* - * - * NOTE. lock_jcr_chain removed, but the message below - * was left until we are sure there are no deadlocks. - * - * We lock the jcr chain here because a good number of the - * callback routines lock the jcr chain. We need to lock - * it here *before* the watchdog lock because the SD message - * thread first locks the jcr chain, then when closing the - * job locks the watchdog chain. If the two threads do not - * lock in the same order, we get a deadlock -- each holds - * the other's needed lock. - */ wd_lock(); walk_list: diff --git a/bacula/src/lib/watchdog.h b/bacula/src/lib/watchdog.h index d6b47e2d47..34a694083f 100644 --- a/bacula/src/lib/watchdog.h +++ b/bacula/src/lib/watchdog.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -32,14 +32,14 @@ enum { #define TIMEOUT_SIGNAL SIGUSR2 struct s_watchdog_t { - bool one_shot; - utime_t interval; - void (*callback)(struct s_watchdog_t *wd); - void (*destructor)(struct s_watchdog_t *wd); - void *data; - /* Private data below - don't touch outside of watchdog.c */ - dlink link; - utime_t next_fire; + bool one_shot; + utime_t interval; + void (*callback)(struct s_watchdog_t *wd); + void (*destructor)(struct s_watchdog_t *wd); + void *data; + /* Private data below - don't touch outside of watchdog.c */ + dlink link; + utime_t next_fire; }; typedef struct s_watchdog_t watchdog_t; diff --git a/bacula/src/lib/worker.c b/bacula/src/lib/worker.c new file mode 100644 index 0000000000..35397fcd2c --- /dev/null +++ b/bacula/src/lib/worker.c @@ -0,0 +1,437 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula worker class. It permits creating a worker thread, + * then sending data via a fifo queue to it. + * + * Kern Sibbald, August 2014 + * + */ + +#define LOCKMGR_COMPLIANT +#include "bacula.h" +#include "worker.h" + +int worker::init(int fifo_size) +{ + int stat; + + if ((stat = pthread_mutex_init(&mutex, NULL)) != 0) { + return stat; + } + if ((stat = pthread_mutex_init(&fmutex, NULL)) != 0) { + pthread_mutex_destroy(&mutex); + return stat; + } + if ((stat = pthread_cond_init(&full_wait, NULL)) != 0) { + pthread_mutex_destroy(&mutex); + pthread_mutex_destroy(&fmutex); + return stat; + } + if ((stat = pthread_cond_init(&empty_wait, NULL)) != 0) { + pthread_cond_destroy(&full_wait); + pthread_mutex_destroy(&mutex); + pthread_mutex_destroy(&fmutex); + return stat; + } + if ((stat = pthread_cond_init(&m_wait, NULL)) != 0) { + pthread_cond_destroy(&empty_wait); + pthread_cond_destroy(&full_wait); + pthread_mutex_destroy(&mutex); + pthread_mutex_destroy(&fmutex); + return stat; + } + valid = WORKER_VALID; + fifo = New(flist(fifo_size)); + fpool = New(alist(fifo_size + 2, false)); + worker_running = false; + set_wait_state(); + return 0; +} + +/* + * Handle cleanup when the lock is released. + */ +static void worker_cleanup(void *arg) +{ + worker *wrk = (worker *)arg; + wrk->release_lock(); +} + + +void worker::release_lock() +{ + pthread_mutex_unlock(&mutex); +} + + +void worker::set_wait_state() +{ + m_state = WORKER_WAIT; +} + +void worker::set_run_state() +{ + if (is_quit_state()) return; + m_state = WORKER_RUN; + if (worker_waiting) { + pthread_cond_signal(&m_wait); + } +} + +void worker::set_quit_state() +{ + P(mutex); + m_state = WORKER_QUIT; + pthread_cond_signal(&m_wait); + pthread_cond_signal(&empty_wait); + V(mutex); +} + + +/* Empty the fifo putting in free pool */ +void worker::discard_queue() +{ + void *item; + + P(mutex); + P(fmutex); + while ((item = fifo->dequeue())) { + fpool->push(item); + } + V(fmutex); + V(mutex); +} + +/* + * Destroy a read/write lock + * + * Returns: 0 on success + * errno on failure + */ +int worker::destroy() +{ + int stat, stat1, stat2, stat3, stat4; + POOLMEM *item; + + m_state = WORKER_QUIT; + pthread_cond_signal(&m_wait); + pthread_cond_signal(&empty_wait); + + P(fmutex); + /* Release free pool */ + while ((item = (POOLMEM *)fpool->pop())) { + free_pool_memory(item); + } + V(fmutex); + fpool->destroy(); + free(fpool); + + /* Release work queue */ + while ((item = (POOLMEM *)fifo->dequeue())) { + free_pool_memory(item); + } + valid = 0; + worker_running = false; + + fifo->destroy(); + free(fifo); + + stat = pthread_mutex_destroy(&mutex); + stat1 = pthread_mutex_destroy(&fmutex); + stat2 = pthread_cond_destroy(&full_wait); + stat3 = pthread_cond_destroy(&empty_wait); + stat4 = pthread_cond_destroy(&m_wait); + if (stat != 0) return stat; + if (stat1 != 0) return stat1; + if (stat2 != 0) return stat2; + if (stat3 != 0) return stat3; + if (stat4 != 0) return stat4; + return 0; +} + + +/* Start the worker thread */ +int worker::start(void *(*auser_sub)(void *), void *auser_ctx) +{ + int stat; + int i; + if (valid != WORKER_VALID) { + return EINVAL; + } + user_sub = auser_sub; + user_ctx = auser_ctx; + if ((stat = pthread_create(&worker_id, NULL, user_sub, this) != 0)) { + return stat; + } + /* Wait for thread to start, but not too long */ + for (i=0; i<100 && !is_running(); i++) { + bmicrosleep(0, 5000); + } + set_run_state(); + return 0; +} + +/* Wait for the worker thread to empty the queue */ +void worker::wait_queue_empty() +{ + if (is_quit_state()) { + return; + } + P(mutex); + while (!empty() && !is_quit_state()) { + pthread_cond_wait(&empty_wait, &mutex); + } + V(mutex); + return; +} + +/* Wait for the main thread to release us */ +void worker::wait() +{ + P(mutex); + pthread_cleanup_push(worker_cleanup, (void *)this); + while (is_wait_state() && !is_quit_state()) { + worker_waiting = true; + pthread_cond_signal(&m_wait); + pthread_cond_wait(&m_wait, &mutex); + } + pthread_cleanup_pop(0); + worker_waiting = false; + V(mutex); +} + +/* Stop the worker thread */ +int worker::stop() +{ + if (valid != WORKER_VALID) { + return EINVAL; + } + m_state = WORKER_QUIT; + pthread_cond_signal(&m_wait); + pthread_cond_signal(&empty_wait); + + if (!pthread_equal(worker_id, pthread_self())) { + pthread_cancel(worker_id); + pthread_join(worker_id, NULL); + } + return 0; +} + + +/* + * Queue an item for the worker thread. Called by main thread. + */ +bool worker::queue(void *item) +{ + bool was_empty = false;; + + if (valid != WORKER_VALID || is_quit_state()) { + return EINVAL; + } + P(mutex); + done = false; + //pthread_cleanup_push(worker_cleanup, (void *)this); + while (full() && !is_quit_state()) { + pthread_cond_wait(&full_wait, &mutex); + } + //pthread_cleanup_pop(0); + /* Maybe this should be worker_running */ + was_empty = empty(); + if (!fifo->queue(item)) { + /* Since we waited for !full this cannot happen */ + V(mutex); + ASSERT2(1, "Fifo queue failed.\n"); + } + if (was_empty) { + pthread_cond_signal(&empty_wait); + } + m_state = WORKER_RUN; + if (worker_waiting) { + pthread_cond_signal(&m_wait); + } + V(mutex); + return 1; +} + +/* + * Wait for work to complete + */ +void worker::finish_work() +{ + P(mutex); + while (!empty() && !is_quit_state()) { + pthread_cond_wait(&empty_wait, &mutex); + } + done = true; /* Tell worker that work is done */ + m_state = WORKER_WAIT; /* force worker into wait state */ + V(mutex); /* pause for state transition */ + if (waiting_on_empty) pthread_cond_signal(&empty_wait); + P(mutex); + /* Wait until worker in wait state */ + while (!worker_waiting && !is_quit_state()) { + if (waiting_on_empty) pthread_cond_signal(&empty_wait); + pthread_cond_wait(&m_wait, &mutex); + } + V(mutex); + discard_queue(); +} + +/* + * Dequeue a work item. Called by worker thread. + */ +void *worker::dequeue() +{ + bool was_full = false;; + void *item = NULL; + + if (valid != WORKER_VALID || done || is_quit_state()) { + return NULL; + } + P(mutex); + //pthread_cleanup_push(worker_cleanup, (void *)this); + while (empty() && !done && !is_quit_state()) { + waiting_on_empty = true; + pthread_cond_wait(&empty_wait, &mutex); + } + waiting_on_empty = false; + //pthread_cleanup_pop(0); + was_full = full(); + item = fifo->dequeue(); + if (was_full) { + pthread_cond_signal(&full_wait); + } + if (empty()) { + pthread_cond_signal(&empty_wait); + } + V(mutex); + return item; +} + +/* + * Pop a free buffer from the list, if one exists. + * Called by main thread to get a free buffer. + * If none exists (NULL returned), it must allocate + * one. + */ +void *worker::pop_free_buffer() +{ + void *free_buf; + + P(fmutex); + free_buf = fpool->pop(); + V(fmutex); + return free_buf; +} + +/* + * Once a work item (buffer) has been processed by the + * worker thread, it will put it on the free buffer list + * (fpool). + */ +void worker::push_free_buffer(void *buf) +{ + P(fmutex); + fpool->push(buf); + V(fmutex); +} + + +//================================================= + +#ifdef TEST_PROGRAM + +void *worker_prog(void *wctx) +{ + POOLMEM *buf; + worker *wrk = (worker *)wctx; + + wrk->set_running(); + + while (!wrk->is_quit_state()) { + if (wrk->is_wait_state()) { + wrk->wait(); + continue; + } + buf = (POOLMEM *)wrk->dequeue(); + if (!buf) { + printf("worker: got null stop\n"); + return NULL; + } + printf("ctx=%lld worker: %s\n", (long long int)wrk->get_ctx(), buf); + wrk->push_free_buffer(buf); + } + printf("worker: asked to stop"); + return NULL; +} + +int main(int argc, char *argv[]) +{ + POOLMEM *buf; + int i; + worker *wrk; + void *ctx; + + wrk = New(worker(10)); + ctx = (void *)1; + wrk->start(worker_prog, ctx); + + for (i=1; i<=40; i++) { + buf = (POOLMEM *)wrk->pop_free_buffer(); + if (!buf) { + buf = get_pool_memory(PM_BSOCK); + printf("Alloc %p\n", buf); + } + sprintf(buf, "This is item %d", i); + wrk->queue(buf); + //printf("back from queue %d\n", i); + } + wrk->wait_queue_empty(); + wrk->set_wait_state(); + printf("======\n"); + for (i=1; i<=5; i++) { + buf = (POOLMEM *)wrk->pop_free_buffer(); + if (!buf) { + buf = get_pool_memory(PM_BSOCK); + printf("Alloc %p\n", buf); + } + sprintf(buf, "This is item %d", i); + wrk->queue(buf); + //printf("back from queue %d\n", i); + } + wrk->set_run_state(); + for (i=6; i<=40; i++) { + buf = (POOLMEM *)wrk->pop_free_buffer(); + if (!buf) { + buf = get_pool_memory(PM_BSOCK); + printf("Alloc %p\n", buf); + } + sprintf(buf, "This is item %d", i); + wrk->queue(buf); + //printf("back from queue %d\n", i); + } + wrk->wait_queue_empty(); + wrk->stop(); + wrk->destroy(); + free(wrk); + + close_memory_pool(); + sm_dump(false); /* test program */ +} +#endif diff --git a/bacula/src/lib/worker.h b/bacula/src/lib/worker.h new file mode 100644 index 0000000000..cea16a4b14 --- /dev/null +++ b/bacula/src/lib/worker.h @@ -0,0 +1,103 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula worker thread class + * + * Kern Sibbald, August 2014 + * + */ + +#ifndef __WORKER_H +#define __WORKER_H 1 + +enum WORKER_STATE { + WORKER_WAIT, + WORKER_RUN, + WORKER_QUIT +}; + +class worker : public SMARTALLOC { +private: + pthread_mutex_t mutex; /* main fifo mutex */ + pthread_mutex_t fmutex; /* free pool mutex */ + pthread_cond_t full_wait; /* wait because full */ + pthread_cond_t empty_wait; /* wait because empty */ + pthread_cond_t m_wait; /* wait state */ + pthread_t worker_id; /* worker's thread id */ + void *(*user_sub)(void *); /* user subroutine */ + void *user_ctx; /* user context */ + flist *fifo; /* work fifo */ + alist *fpool; /* free pool */ + int valid; /* set when valid */ + WORKER_STATE m_state; /* worker state */ + bool worker_running; /* set when worker running */ + bool worker_waiting; /* set when worker is waiting */ + bool done; /* work done */ + bool waiting_on_empty; /* hung waiting on empty queue */ + + +public: + worker(int fifo_size = 10); + ~worker(); + int init(int fifo_size = 10); + int destroy(); + + bool queue(void *item); + void *dequeue(); + int start(void *(*sub)(void *), void *wctx); /* Start worker */ + void wait_queue_empty(); /* Main thread wait for fifo to be emptied */ + void discard_queue(); /* Discard the fifo queue */ + int stop(); /* Stop worker */ + void wait(); /* Wait for main thread to release us */ + void set_wait_state(); + void set_run_state(); + void set_quit_state(); + void finish_work(); + + void *pop_free_buffer(); + void push_free_buffer(void *buf); + + inline void set_running() { worker_running = true; }; + inline bool is_running() const { return worker_running; }; + inline void *get_ctx() const { return user_ctx; }; + + void release_lock(); /* Cleanup release lock */ + + inline bool empty() const { return fifo->empty(); }; + inline bool full() const { return fifo->full(); }; + inline int size() const { return fifo->size(); }; + inline bool is_quit_state() const { return m_state == WORKER_QUIT; }; + inline bool is_wait_state() const { return m_state == WORKER_WAIT; }; +}; + + +inline worker::worker(int fifo_size) +{ + init(fifo_size); +} + +inline worker::~worker() +{ + destroy(); +} + + +#define WORKER_VALID 0xfadbec + +#endif /* __WORKER_H */ diff --git a/bacula/src/lib/workq.c b/bacula/src/lib/workq.c index bfed633a8d..115db7ebdc 100644 --- a/bacula/src/lib/workq.c +++ b/bacula/src/lib/workq.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -41,6 +41,12 @@ * Emsg1(M_ABORT, 0, "Could not add job to work queue: ERR=%s\n", be.bstrerror(errno)); * } * + * Wait for all queued work to be completed + * if ((stat = workq_wait_idle(&job_wq, (void *)jcr)) != 0) { + * berrno be; + * Emsg1(M_ABORT, 0, "Could not wait for idle: ERR=%s\n", be.bstrerror(errno)); + * } + * * Terminate the queue * workq_destroy(workq_t *wq); * @@ -78,17 +84,24 @@ int workq_init(workq_t *wq, int threads, void *(*engine)(void *arg)) pthread_attr_destroy(&wq->attr); return stat; } + if ((stat = pthread_cond_init(&wq->idle, NULL)) != 0) { + pthread_mutex_destroy(&wq->mutex); + pthread_attr_destroy(&wq->attr); + pthread_cond_destroy(&wq->work); + return stat; + } wq->quit = 0; wq->first = wq->last = NULL; wq->max_workers = threads; /* max threads to create */ wq->num_workers = 0; /* no threads yet */ + wq->num_running = 0; /* no running threads */ wq->idle_workers = 0; /* no idle threads */ wq->engine = engine; /* routine to run */ wq->valid = WORKQ_VALID; return 0; } -/* +/* [B * Destroy a work queue * * Returns: 0 on success @@ -96,7 +109,7 @@ int workq_init(workq_t *wq, int threads, void *(*engine)(void *arg)) */ int workq_destroy(workq_t *wq) { - int stat, stat1, stat2; + int stat, stat1, stat2, stat3; if (wq->valid != WORKQ_VALID) { return EINVAL; @@ -126,9 +139,41 @@ int workq_destroy(workq_t *wq) stat = pthread_mutex_destroy(&wq->mutex); stat1 = pthread_cond_destroy(&wq->work); stat2 = pthread_attr_destroy(&wq->attr); - return (stat != 0 ? stat : (stat1 != 0 ? stat1 : stat2)); + stat3 = pthread_cond_destroy(&wq->idle); + if (stat != 0) return stat; + if (stat1 != 0) return stat1; + if (stat2 != 0) return stat2; + if (stat3 != 0) return stat3; + return 0; } +/* + * Wait for work to terminate + * + * Returns: 0 on success + * errno on failure + */ +int workq_wait_idle(workq_t *wq) +{ + int stat; + + if (wq->valid != WORKQ_VALID) { + return EINVAL; + } + P(wq->mutex); + + /* While there is work, wait */ + while (wq->num_running || wq->first != NULL) { + if ((stat = pthread_cond_wait(&wq->idle, &wq->mutex)) != 0) { + V(wq->mutex); + return stat; + } + } + V(wq->mutex); + return 0; +} + + /* * Add work to a queue @@ -320,6 +365,7 @@ void *workq_server(void *arg) if (wq->last == we) { wq->last = NULL; } + wq->num_running++; V(wq->mutex); /* Call user's routine here */ Dmsg0(1400, "Calling user engine.\n"); @@ -328,8 +374,12 @@ void *workq_server(void *arg) free(we); /* release work entry */ Dmsg0(1400, "relock mutex\n"); P(wq->mutex); + wq->num_running--; Dmsg0(1400, "Done lock mutex\n"); } + if (wq->first == NULL && !wq->num_running) { + pthread_cond_broadcast(&wq->idle); + } /* * If no more work request, and we are asked to quit, then do it */ @@ -364,3 +414,109 @@ void *workq_server(void *arg) Dmsg0(1400, "End workq_server\n"); return NULL; } + + +//================================================= +#ifdef TEST_PROGRAM + +#define TEST_SLEEP_TIME_IN_SECONDS 3 +#define TEST_MAX_NUM_WORKERS 5 +#define TEST_NUM_WORKS 10 + + +void *callback(void *ctx) +{ + JCR* jcr = (JCR*)ctx; + + if (jcr) + { + Jmsg1(jcr, M_INFO, 0, _("workq_test: thread %d : now starting work....\n"), (int)pthread_self()); + sleep(TEST_SLEEP_TIME_IN_SECONDS); + Jmsg1(jcr, M_INFO, 0, _("workq_test: thread %d : ...work completed.\n"), (int)pthread_self()); + } + return NULL; +} + + +char *configfile = NULL; +//STORES *me = NULL; /* our Global resource */ +bool forge_on = false; /* proceed inspite of I/O errors */ +pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; + +int main (int argc, char *argv[]) +{ + pthread_attr_t attr; + + void * start_heap = sbrk(0); + (void)start_heap; + + setlocale(LC_ALL, ""); + bindtextdomain("bacula", LOCALEDIR); + textdomain("bacula"); + init_stack_dump(); + my_name_is(argc, argv, "workq_test"); + init_msg(NULL, NULL); + daemon_start_time = time(NULL); + set_thread_concurrency(150); + lmgr_init_thread(); /* initialize the lockmanager stack */ + pthread_attr_init(&attr); + + int stat(-1); + berrno be; + + workq_t queue; + /* Start work queues */ + if ((stat = workq_init(&queue, TEST_MAX_NUM_WORKERS, callback)) != 0) + { + be.set_errno(stat); + Emsg1(M_ABORT, 0, _("Could not init work queue: ERR=%s\n"), be.bstrerror()); + } + + /* job1 is created and pseudo-submits some work to the work queue*/ + JCR *jcr1 = new_jcr(sizeof(JCR), NULL); + jcr1->JobId = 1; + workq_ele_t * ret(0); + for (int w=0; wJobId = 2; + for (int w=0; wDebugMessage(context, __FILE__, __LINE__, level, __VA_ARGS__ ) #define Jmsg(context, type, ...) bfuncs->JobMessage(context, __FILE__, __LINE__, type, 0, __VA_ARGS__ ) #ifdef USE_CMD_PARSER -/* from lib/scan.c */ -extern int parse_args(POOLMEM *cmd, POOLMEM **args, int *argc, - char **argk, char **argv, int max_args); -extern int parse_args_only(POOLMEM *cmd, POOLMEM **args, int *argc, - char **argk, char **argv, int max_args); +#include "lib/cmd_parser.h" +#endif /* USE_CMD_PARSER */ -class cmd_parser { -public: - POOLMEM *args; - POOLMEM *cmd; /* plugin command line */ - POOLMEM *org; /* original command line */ - - char **argk; /* Argument keywords */ - char **argv; /* Argument values */ - int argc; /* Number of arguments */ - int max_cmd; /* Max number of arguments */ - bool use_name; /* Search for : */ - - cmd_parser() { - org = get_pool_memory(PM_FNAME); - args = get_pool_memory(PM_FNAME); - cmd = get_pool_memory(PM_FNAME); - *args = *org = *cmd = 0; - argc = 0; - use_name = true; - max_cmd = MAX_CMD_ARGS; - argk = argv = NULL; - }; - - virtual ~cmd_parser() { - free_pool_memory(org); - free_pool_memory(cmd); - free_pool_memory(args); - if (argk) { - free(argk); - } - if (argv) { - free(argv); +#ifdef USE_ADD_DRIVE +/* Keep drive letters for windows vss snapshot */ +static void add_drive(char *drives, int *nCount, char *fname) { + if (strlen(fname) >= 2 && B_ISALPHA(fname[0]) && fname[1] == ':') { + /* always add in uppercase */ + char ch = toupper(fname[0]); + /* if not found in string, add drive letter */ + if (!strchr(drives,ch)) { + drives[*nCount] = ch; + drives[*nCount+1] = 0; + (*nCount)++; + } + } +} + +/* Copy our drive list to Bacula core list */ +static void copy_drives(char *drives, char *dest) { + int last = strlen(dest); /* dest is 27 bytes long */ + for (char *p = drives; *p && last < 26; p++) { + if (!strchr(dest, *p)) { + dest[last++] = *p; + dest[last] = 0; } } +} +#endif /* USE_ADD_DRIVE */ - /* - * Given a single keyword, find it in the argument list, but - * it must have a value - * Returns: -1 if not found or no value - * list index (base 0) on success - */ - int find_arg_with_value(const char *keyword) +#endif /* ! PCOMMON_H */ + +/* Check if the bacula version is enterprise */ +#ifndef check_beef +# define check_beef(ctx, ret) \ + do { \ + const char *v; \ + bfuncs->getBaculaValue(ctx, bVarVersion, (void *)&v); \ + if (v[0] == '6' || v[0] == '8') { \ + *(ret) = true; \ + } else { \ + *(ret) = false; \ + } \ + } while (0) + +#endif /* check_beef */ + +#ifdef USE_JOB_LIST + +/* This class is used to store locally the job history, you can attach data + * to it such as snapshot names + * !!! Don't forget that this file may be deleted by the user. !!! + */ + +class joblist: public SMARTALLOC +{ +private: + bpContext *ctx; + +public: + char level; /* level of the job */ + + char base[MAX_NAME_LENGTH]; /* base name */ + char key[MAX_NAME_LENGTH]; /* group of backup */ + char name[MAX_NAME_LENGTH]; /* job name */ + char prev[MAX_NAME_LENGTH]; /* based on jobname */ + char root[MAX_NAME_LENGTH]; /* root of this branch */ + char rootdiff[MAX_NAME_LENGTH]; /* root of diff if any */ + + btime_t job_time; /* job time */ + + void init() { + level = 0; + job_time = 0; + *key = *name = *prev = *root = *rootdiff = 0; + set_base("jobs.dat"); + ctx = NULL; + } + + void set_base(const char *b) { + strncpy(base, b, sizeof(base)); + } + + joblist(bpContext *actx) { init(); ctx = actx; } + + joblist(bpContext *actx, + const char *akey, + const char *jobname, + const char *prevjobname, + char joblevel) { - for (int i=1; i dlen) { /* we probably have a key */ + int start = l - dlen; + if (name[start] == '.' && + B_ISDIGIT(name[start + 1]) && // 2 + B_ISDIGIT(name[start + 2]) && // 0 + B_ISDIGIT(name[start + 3]) && // 1 + B_ISDIGIT(name[start + 4]) && // 2 + name[start + 5] == '-' && // - + B_ISDIGIT(name[start + 6]) && // 0 + B_ISDIGIT(name[start + 7])) // 7 + { + bstrncpy(key, name, start + 1); + Dmsg(ctx, dbglvl+100, "key is %s from jobname %s\n", key, name); + return true; } } - return -1; + Dmsg(ctx, dbglvl+100, "Unable to get key from jobname %s\n", name); + return false; } - /* - * Build args, argc, argk from Plugin Restore|Backup command - */ + bool find_job(const char *name, POOLMEM **data=NULL); /* set root, job_time */ + bool find_root_job(); + void store_job(char *data); + void prune_jobs(char *build_cmd(void *arg, const char *data, const char *job), + void *arg, alist *jobs); +}; - bRC parse_cmd(const char *line) - { - char *a; - int nbequal = 0; - if (!line || *line == '\0') { - return bRC_Error; - } +static pthread_mutex_t joblist_mutex = PTHREAD_MUTEX_INITIALIZER; + +bool joblist::find_job(const char *name, POOLMEM **data) +{ + BFILE fp; + FILE *f; + POOLMEM *tmp; + char buf[1024]; + char curkey[MAX_NAME_LENGTH]; /* key */ + char curjobname[MAX_NAME_LENGTH]; /* jobname */ + char prevjob[MAX_NAME_LENGTH]; /* last jobname */ + char rootjob[MAX_NAME_LENGTH]; /* root jobname */ + char t[MAX_NAME_LENGTH]; + char curlevel; + bool ok=false; + + *root = 0; + job_time = 0; + *rootdiff = 0; + + binit(&fp); + set_portable_backup(&fp); + + tmp = get_pool_memory(PM_FNAME); + Mmsg(tmp, "%s/%s", working, base); + + P(joblist_mutex); + if (bopen(&fp, tmp, O_RDONLY, 0) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to open job database. " + "Can't open %s for reading ERR=%s\n", + tmp, be.bstrerror(errno)); + goto bail_out; + } - /* Same command line that before */ - if (!strcmp(line, org)) { - return bRC_OK; + f = fdopen(fp.fid, "r"); + if (!f) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to open job database. ERR=%s\n", + be.bstrerror(errno)); + goto bail_out; + } + + while (!ok && fgets(buf, sizeof(buf), f) != NULL) { + *curkey = *curjobname = *rootjob = *prevjob = 0; + + Dmsg(ctx, dbglvl+100, "line = [%s]\n", buf); + + if (sscanf(buf, "time=%60s level=%c key=%127s name=%127s root=%127s prev=%127s", + t, &curlevel, curkey, curjobname, rootjob, prevjob) != 6) { + + if (sscanf(buf, "time=%60s level=F key=%127s name=%127s", + t, curkey, curjobname) != 3) { + Dmsg(ctx, dbglvl+100, "Bad line l=[%s]\n", buf); + continue; + } } - /* - * line = delta:minsize=10002 param1=xxx - * | backup command - */ - pm_strcpy(org, line); - pm_strcpy(cmd, line); - - if (use_name) { - if ((a = strchr(cmd, ':')) != NULL) { - *a = ' '; /* replace : by ' ' for command line processing */ + if (strcmp(name, curjobname) == 0 && + strcmp(key, curkey) == 0) + { + job_time = str_to_uint64(t); + bstrncpy(root, rootjob, MAX_NAME_LENGTH); + if (curlevel == 'D') { + bstrncpy(rootdiff, curjobname, MAX_NAME_LENGTH); + } + + if (data) { + pm_strcpy(data, strstr(buf, " vol=") + 5); + strip_trailing_newline(*data); + unbash_spaces(*data); + } + + ok = true; + Dmsg(ctx, dbglvl+100, "Found job root %s -> %s -> %s\n", + rootdiff, root, curjobname); + } + } + + fclose(f); + +bail_out: + V(joblist_mutex); + free_pool_memory(tmp); + return ok; + +} + +/* Find the root job for the current job */ +bool joblist::find_root_job() +{ + BFILE fp; + FILE *f; + POOLMEM *tmp; + char buf[1024]; + char curkey[MAX_NAME_LENGTH]; /* key */ + char curjobname[MAX_NAME_LENGTH]; /* jobname */ + char prevjob[MAX_NAME_LENGTH]; /* last jobname */ + char rootjob[MAX_NAME_LENGTH]; /* root jobname */ + char t[MAX_NAME_LENGTH]; + char curlevel; + bool ok=false; + + *root = 0; + job_time = 0; + + if (level == 'F') { + bstrncpy(root, name, MAX_NAME_LENGTH); + return true; + } + + binit(&fp); + set_portable_backup(&fp); + + tmp = get_pool_memory(PM_FNAME); + Mmsg(tmp, "%s/%s", working, base); + + P(joblist_mutex); + if (bopen(&fp, tmp, O_RDONLY, 0) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to prune previous jobs. " + "Can't open %s for reading ERR=%s\n", + tmp, be.bstrerror(errno)); + goto bail_out; + } + + f = fdopen(fp.fid, "r"); + if (!f) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to prune previous jobs. ERR=%s\n", + be.bstrerror(errno)); + goto bail_out; + } + + while (!ok && fgets(buf, sizeof(buf), f) != NULL) { + *curkey = *curjobname = *rootjob = *prevjob = 0; + + Dmsg(ctx, dbglvl+100, "line = [%s]\n", buf); + + if (sscanf(buf, "time=%60s level=%c key=%127s name=%127s root=%127s prev=%127s", + t, &curlevel, curkey, curjobname, rootjob, prevjob) != 6) { + + if (sscanf(buf, "time=%60s level=F key=%127s name=%127s", + t, curkey, curjobname) == 3) { + bstrncpy(rootjob, curjobname, MAX_NAME_LENGTH); + *prevjob = 0; + curlevel = 'F'; + } else { - return bRC_Error; + Dmsg(ctx, dbglvl+100, "Bad line l=[%s]\n", buf); + continue; } } + + if (strcmp(key, curkey) == 0 && + strcmp(prev, curjobname) == 0) + { + bstrncpy(root, rootjob, MAX_NAME_LENGTH); + + if (curlevel == 'D') { + bstrncpy(rootdiff, curjobname, MAX_NAME_LENGTH); + } + ok = true; + Dmsg(ctx, dbglvl+100, "Found job root %s -> %s -> %s\n", + rootdiff, root, curjobname); + } + } + + fclose(f); - for (char *p = cmd; *p ; p++) { - if (*p == '=') { - nbequal++; +bail_out: + V(joblist_mutex); + free_pool_memory(tmp); + return true; +} + +/* Store the current job in the jobs.dat for a specific data list */ +void joblist::store_job(char *data) +{ + BFILE fp; + int l; + POOLMEM *tmp = NULL; + btime_t now; + + /* Not initialized, no need to store jobs */ + if (*name == 0 || !level) { + Dmsg(ctx, dbglvl+100, "store_job fail name=%s level=%d\n", name, level); + return; + } + + find_root_job(); + + binit(&fp); + set_portable_backup(&fp); + + P(joblist_mutex); + + tmp = get_pool_memory(PM_FNAME); + Mmsg(tmp, "%s/%s", working, base); + if (bopen(&fp, tmp, O_WRONLY|O_CREAT|O_APPEND, 0600) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to update the job history. ERR=%s\n", + be.bstrerror(errno)); + goto bail_out; + } + + now = time(NULL); + + bash_spaces(data); + + if (level == 'F') { + l = Mmsg(tmp, "time=%lld level=%c key=%s name=%s vollen=%d vol=%s\n", + now, level, key, name, strlen(data), data); + + } else { + l = Mmsg(tmp, "time=%lld level=%c key=%s name=%s root=%s prev=%s vollen=%d vol=%s\n", + now, level, key, name, root, prev, strlen(data), data); + } + + if (bwrite(&fp, tmp, l) != l) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to update the job history. ERR=%s\n", + be.bstrerror(errno)); + } + + bclose(&fp); + +bail_out: + V(joblist_mutex); + free_pool_memory(tmp); +} + +/* Prune jobs at the end of the job, this function can generate commands + * in order to cleanup something + */ +void joblist::prune_jobs(char *build_cmd(void *arg, const char *data, const char *job), + void *arg, alist *jobs) +{ + BFILE fp, fpout; + FILE *f=NULL; + POOLMEM *tmp; + POOLMEM *tmpout; + POOLMEM *data; + POOLMEM *buf; + char curkey[MAX_NAME_LENGTH]; /* key */ + char jobname[MAX_NAME_LENGTH]; /* jobname */ + char prevjob[MAX_NAME_LENGTH]; /* last jobname */ + char rootjob[MAX_NAME_LENGTH]; /* root jobname */ + char t[MAX_NAME_LENGTH]; + uint32_t datalen; + char curlevel; + bool keep; + bool ok=false; + int count=0, len; + + /* In Incremental, it means that the previous Full/Diff is well terminated */ + if (level != 'I') { + return; + } + + find_root_job(); + + binit(&fp); + set_portable_backup(&fp); + + binit(&fpout); + set_portable_backup(&fpout); + + tmp = get_pool_memory(PM_FNAME); + Mmsg(tmp, "%s/%s", working, base); + + tmpout = get_pool_memory(PM_FNAME); + Mmsg(tmpout, "%s/%s.swap", working, base); + + buf = get_pool_memory(PM_FNAME); + data = get_pool_memory(PM_FNAME); + *buf = *data = 0; + + P(joblist_mutex); + if (bopen(&fp, tmp, O_RDONLY, 0) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to prune previous jobs. " + "Can't open %s for reading ERR=%s\n", + tmp, be.bstrerror(errno)); + goto bail_out; + } + if (bopen(&fpout, tmpout, O_CREAT|O_WRONLY, 0600) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to prune previous jobs. " + "Can't open %s for writing ERR=%s\n", + tmpout, be.bstrerror(errno)); + goto bail_out; + } + + f = fdopen(fp.fid, "r"); /* we use fgets from open() */ + if (!f) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to prune previous jobs. ERR=%s\n", + be.bstrerror(errno)); + goto bail_out; + } + + while (fgets(buf, sizeof_pool_memory(buf), f) != NULL) { + *data = *curkey = *jobname = *rootjob = *prevjob = 0; + keep = false; + datalen = 0; + + len = strlen(buf); + if (len > 0 && buf[len -1] != '\n') { + /* The line is larger than the buffer, we need to capture the rest */ + bool ok=false; + while (!ok) { + Dmsg(ctx, dbglvl+100, "Reading extra 1024 bytes, len=%d\n", len); + buf = check_pool_memory_size(buf, sizeof_pool_memory(buf) + 1024); + if (fgets(buf + len, 1023, f) == NULL) { + ok = true; + } + len = strlen(buf); + if (buf[len - 1] == '\n') { + ok = true; + } + if (len > 32000) { /* sanity check */ + ok = true; + } } } - if (argk) { - free(argk); + /* We don't capture the vol list, because our sscanf is limited to 1000 bytes */ + if (sscanf(buf, "time=%60s level=%c key=%127s name=%127s root=%127s prev=%127s vollen=%d vol=", + t, &curlevel, curkey, jobname, rootjob, prevjob, &datalen) != 7) { + + if (sscanf(buf, "time=%60s level=F key=%127s name=%127s vollen=%d vol=", + t, curkey, jobname, &datalen) == 4) { + *rootdiff = *rootjob = *prevjob = 0; + curlevel = 'F'; + + } else { + Dmsg(ctx, dbglvl+100, "Bad line l=[%s]\n", buf); + keep = true; + } } - if (argv) { - free(argv); + + if (!keep) { + pm_strcpy(data, strstr(buf, " vol=") + 5); + strip_trailing_newline(data); + unbash_spaces(data); + + if (datalen != strlen(data)) { + Dmsg(ctx, dbglvl+100, "Bad data line datalen != strlen(data) %d != %d\n", datalen, strlen(data)); + Dmsg(ctx, dbglvl+100, "v=[%s]\n", data); + } } - max_cmd = MAX(nbequal, MAX_CMD_ARGS) + 1; - argk = (char **) malloc(sizeof(char **) * max_cmd); - argv = (char **) malloc(sizeof(char **) * max_cmd); - - parse_args(cmd, &args, &argc, argk, argv, max_cmd); + if (!keep && + (strcmp(key, curkey) != 0 || + strcmp(name, jobname) == 0 || + strcmp(prev, jobname) == 0 || + strcmp(root, jobname) == 0 || + strcmp(rootdiff, jobname) == 0)) + { + keep = true; + } - return bRC_OK; + if (keep) { + if (bwrite(&fpout, buf, len) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to update the job history. ERR=%s\n", + be.bstrerror(errno)); + goto bail_out; + } + + } else if (build_cmd) { + count++; + Dmsg(ctx, dbglvl+100, "Can prune jobname %s\n", jobname); + + char *p2 = data; + for(char *p = data; *p; p++) { + if (*p == ',') { + *p = 0; + jobs->append(bstrdup(build_cmd(arg, p2, jobname))); + p2 = p + 1 ; + } + } + jobs->append(bstrdup(build_cmd(arg, p2, jobname))); + + } else if (jobs) { + jobs->append(bstrdup(data)); + } } -}; -#endif /* USE_CMD_PARSER */ + ok = true; -#ifdef USE_ADD_DRIVE -/* Keep drive letters for windows vss snapshot */ -static void add_drive(char *drives, int *nCount, char *fname) { - if (strlen(fname) >= 2 && B_ISALPHA(fname[0]) && fname[1] == ':') { - /* always add in uppercase */ - char ch = toupper(fname[0]); - /* if not found in string, add drive letter */ - if (!strchr(drives,ch)) { - drives[*nCount] = ch; - drives[*nCount+1] = 0; - (*nCount)++; - } +bail_out: + if (f) { + fclose(f); + } + if (is_bopen(&fpout)) { + bclose(&fpout); } -} -/* Copy our drive list to Bacula core list */ -static void copy_drives(char *drives, char *dest) { - int last = strlen(dest); /* dest is 27 bytes long */ - for (char *p = drives; *p && last < 26; p++) { - if (!strchr(dest, *p)) { - dest[last++] = *p; - dest[last] = 0; + /* We can switch the file */ + if (ok) { + unlink(tmp); + + if (rename(tmpout, tmp) < 0) { + berrno be; + Jmsg(ctx, M_ERROR, "Unable to update the job history. ERR=%s\n", + be.bstrerror(errno)); } } + + V(joblist_mutex); + free_pool_memory(tmp); + free_pool_memory(tmpout); + free_pool_memory(data); + free_pool_memory(buf); + + Dmsg(ctx, dbglvl+100, "Pruning %d jobs\n", count); } -#endif /* USE_ADD_DRIVE */ -#endif /* ! PCOMMON_H */ + +#endif /* ! USE_JOB_LIST */ + diff --git a/bacula/src/plugins/sd/example-plugin-sd.c b/bacula/src/plugins/sd/example-plugin-sd.c index 21b674a286..742f3d30e9 100644 --- a/bacula/src/plugins/sd/example-plugin-sd.c +++ b/bacula/src/plugins/sd/example-plugin-sd.c @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2007-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Sample Storage daemon Plugin program * * Kern Sibbald, October 2007 - * */ #include "bacula.h" /* General Bacula headers */ #include "stored.h" /* Pull in storage daemon headers */ diff --git a/bacula/src/plugins/sd/main.c b/bacula/src/plugins/sd/main.c index 4be52758c6..2b97ce8ed6 100644 --- a/bacula/src/plugins/sd/main.c +++ b/bacula/src/plugins/sd/main.c @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2007-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. diff --git a/bacula/src/qt-console/bat.h b/bacula/src/qt-console/bat.h index 5280d189c1..ad26ed0a45 100644 --- a/bacula/src/qt-console/bat.h +++ b/bacula/src/qt-console/bat.h @@ -44,6 +44,8 @@ #include #include #include "bacula.h" + +#ifndef TRAY_MONITOR #include "mainwin.h" #include "bat_conf.h" #include "jcr.h" @@ -51,6 +53,7 @@ extern MainWin *mainWin; extern QApplication *app; +#endif bool isWin32Path(QString &fullPath); diff --git a/bacula/src/qt-console/bat.pro.in b/bacula/src/qt-console/bat.pro.in index 7c43f13681..d1bbdb8629 100644 --- a/bacula/src/qt-console/bat.pro.in +++ b/bacula/src/qt-console/bat.pro.in @@ -8,6 +8,8 @@ # CONFIG += qt debug @QWT@ +datarootdir = @datarootdir@ + bins.path = /$(DESTDIR)@sbindir@ bins.files = bat confs.path = /$(DESTDIR)@sysconfdir@ diff --git a/bacula/src/qt-console/bat_conf.cpp b/bacula/src/qt-console/bat_conf.cpp index c240ed6ca0..ce2ab36fd4 100644 --- a/bacula/src/qt-console/bat_conf.cpp +++ b/bacula/src/qt-console/bat_conf.cpp @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -48,8 +48,7 @@ */ int32_t r_first = R_FIRST; int32_t r_last = R_LAST; -static RES *sres_head[R_LAST - R_FIRST + 1]; -RES **res_head = sres_head; +RES_HEAD **res_head; /* Forward referenced subroutines */ @@ -99,6 +98,7 @@ static RES_ITEM con_items[] = { {"tlskey", store_dir, ITEM(con_res.tls_keyfile), 0, 0, 0}, {"heartbeatinterval", store_time, ITEM(con_res.heartbeat_interval), 0, ITEM_DEFAULT, 5 * 60}, {"director", store_str, ITEM(con_res.director), 0, 0, 0}, + {"CommCompression", store_bool, ITEM(con_res.comm_compression), 0, ITEM_DEFAULT, true}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -125,6 +125,7 @@ RES_TABLE resources[] = { /* Dump contents of resource */ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, ...), void *sock) { + RES *next; URES *res = (URES *)ares; bool recurse = true; @@ -151,8 +152,11 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, default: printf(_("Unknown resource type %d\n"), type); } - if (recurse && res->dir_res.hdr.next) { - dump_resource(type, res->dir_res.hdr.next, sendit, sock); + if (recurse) { + next = GetNextRes(0, (RES *)res); + if (next) { + dump_resource(type, next, sendit, sock); + } } } @@ -165,14 +169,12 @@ void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, */ void free_resource(RES *sres, int type) { - RES *nres; URES *res = (URES *)sres; if (res == NULL) return; /* common stuff -- free the resource name */ - nres = (RES *)res->dir_res.hdr.next; if (res->dir_res.hdr.name) { free(res->dir_res.hdr.name); } @@ -236,18 +238,14 @@ void free_resource(RES *sres, int type) if (res) { free(res); } - if (nres) { - free_resource(nres, type); - } } /* Save the new resource by chaining it into the head list for * the resource. If this is pass 2, we update any resource * pointers (currently only in the Job resource). */ -void save_resource(int type, RES_ITEM *items, int pass) +bool save_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { - URES *res; int rindex = type - r_first; int i, size = 0; int error = 0; @@ -257,10 +255,11 @@ void save_resource(int type, RES_ITEM *items, int pass) */ for (i=0; items[i].name; i++) { if (items[i].flags & ITEM_REQUIRED) { - if (!bit_is_set(i, res_all.dir_res.hdr.item_present)) { - Emsg2(M_ABORT, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + if (!bit_is_set(i, res_all.dir_res.hdr.item_present)) { + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), items[i].name, resources[rindex].name); - } + return false; + } } } @@ -295,7 +294,7 @@ void save_resource(int type, RES_ITEM *items, int pass) free(res_all.dir_res.hdr.desc); res_all.dir_res.hdr.desc = NULL; } - return; + return true; } /* The following code is only executed during pass 1 */ @@ -316,31 +315,16 @@ void save_resource(int type, RES_ITEM *items, int pass) } /* Common */ if (!error) { - res = (URES *)malloc(size); - memcpy(res, &res_all, size); - if (!res_head[rindex]) { - res_head[rindex] = (RES *)res; /* store first entry */ - } else { - RES *next, *last; - /* Add new res to end of chain */ - for (last=next=res_head[rindex]; next; next=next->next) { - last = next; - if (strcmp(next->name, res->dir_res.hdr.name) == 0) { - Emsg2(M_ERROR_TERM, 0, - _("Attempt to define second \"%s\" resource named \"%s\" is not permitted.\n"), - resources[rindex].name, res->dir_res.hdr.name); - } - } - last->next = (RES *)res; - Dmsg2(90, "Inserting %s res: %s\n", res_to_str(type), - res->dir_res.hdr.name); + if (!config->insert_res(rindex, size)) { + return false; } } + return true; } bool parse_bat_config(CONFIG *config, const char *configfile, int exit_code) { config->init(configfile, NULL, exit_code, (void *)&res_all, res_all_size, - r_first, r_last, resources, res_head); + r_first, r_last, resources, &res_head); return config->parse_config(); } diff --git a/bacula/src/qt-console/bat_conf.h b/bacula/src/qt-console/bat_conf.h index 7fa7a19951..7321f3dddd 100644 --- a/bacula/src/qt-console/bat_conf.h +++ b/bacula/src/qt-console/bat_conf.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,13 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * Bacula Adminstration Tool (bat) + * Bacula Adminstration Tool (bat) * * Kern Sibbald, March 2002 */ @@ -83,6 +83,7 @@ class CONRES { public: RES hdr; char *password; /* UA server password */ + bool comm_compression; /* Enable comm line compression */ bool tls_authenticate; /* Authenticate with tls */ bool tls_enable; /* Enable TLS on all connections */ bool tls_require; /* Require TLS on all connections */ diff --git a/bacula/src/qt-console/bcomm/dircomm_auth.cpp b/bacula/src/qt-console/bcomm/dircomm_auth.cpp index 8c24ca31b7..9da403e87d 100644 --- a/bacula/src/qt-console/bcomm/dircomm_auth.cpp +++ b/bacula/src/qt-console/bcomm/dircomm_auth.cpp @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,12 +11,11 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ - /* * * Bacula UA authentication. Provides authentication with @@ -43,6 +42,7 @@ static char hello[] = "Hello %s calling %d\n"; /* Response from Director */ static char oldOKhello[] = "1000 OK:"; static char newOKhello[] = "1000 OK: %d"; +static char FDOKhello[] = "2000 OK Hello %d"; /* Forward referenced functions */ @@ -153,13 +153,23 @@ bool DirComm::authenticate_director(JCR *jcr, DIRRES *director, CONRES *cons, dir->stop_timer(); Dmsg1(10, "msg); - if (strncmp(dir->msg, oldOKhello, sizeof(oldOKhello)-1) != 0) { + if (strncmp(dir->msg, oldOKhello, sizeof(oldOKhello)-1) == 0) { + /* If Dir version exists, get it */ + sscanf(dir->msg, newOKhello, &dir_version); + + /* We do not check the last %d */ + } else if (strncmp(dir->msg, FDOKhello, sizeof(FDOKhello)-3) == 0) { + sscanf(dir->msg, FDOKhello, &dir_version); + // TODO: Keep somewhere that we need a proxy command, or run it directly? + } else { bsnprintf(errmsg, errmsg_len, _("Director at \"%s:%d\" rejected Hello command\n"), - dir->host(), dir->port()); + dir->host(), dir->port()); return false; - } else { - /* If Dir version exists, get it */ - sscanf(dir->msg, newOKhello, &dir_version); + } + + /* Turn on compression for newer Directors */ + if (dir_version >= 1 && (!cons || cons->comm_compression)) { + dir->set_compress(); } if (m_conn == 0) { diff --git a/bacula/src/qt-console/help/restore.html b/bacula/src/qt-console/help/restore.html index df4ab7b627..cbae957081 100644 --- a/bacula/src/qt-console/help/restore.html +++ b/bacula/src/qt-console/help/restore.html @@ -132,25 +132,6 @@ noticed that the performance can begin to be quite slow. A way to improve the response time of the database server is to add indexes that will assist a couple of the specific queries that are made. -

For postgresql add 2 indexes with the following commands in psql: -
CREATE INDEX file_filenameid_jobid ON file USING btree (filenameid, -jobid); -
CREATE INDEX file_pathid_idx ON file USING btree (pathid); - -

For mysql add 2 indexes with the following commands in mysql: -
CREATE INDEX file_filenameid_jobid ON File (FilenameId, JobId); -
CREATE INDEX file_pathid_idx ON File (PathId); - -

There is one way to make the first of those two indexes perform just a -little better. It is to create a partial index. First, at least one backup -must be in the database that has at least one directory. Then in psql or mysql -perform the command: -
SELECT FilenameId FROM Filename WHERE name=''; -
Use the results of this command and replace for XXX in the following command: -
CREATE INDEX file_filenameid_jobid2 ON File (JobId) WHERE FilenameId=XXX; -
This index will use less disk space and will perform better. Don't forget to -remove the index it replaces, file_filenameid_jobid. -

If you have sqlite and would be willing to test out the creation of these indexes to see if they work, please let me know the commands. diff --git a/bacula/src/qt-console/images/ajax-loader-big.gif b/bacula/src/qt-console/images/ajax-loader-big.gif new file mode 100644 index 0000000000000000000000000000000000000000..3288d1035d70bb86517e2c233f1a904e41f06b29 GIT binary patch literal 3208 zcmc(iX;4#H9>pJdFE7h`I{IF)0|5<6L}(j=N}5%L009EB2nYfyF)E0PvIqo$u!IC; z4PgyY5|S9AEh38G)(9eq4TbH7_UHg@yWrlIJ$6smIADL7s^P;_O;ykRc9soXl`UC*LwQJXkii*0rx|*7rI2=x7WaRkx_~XZqFJ8R3c=2Kg zf@aSAv8+BJ8+^hyay>(QR@t*blbKzsf0}bscEqRc5Hd3o(-N5RyW=zWB*zQw6Zh>* z2CROCDAbu#D`)S|J_o(lL9Yn3l*+8RdiRD_>iNz$#_IAzCna&Wl5 zSF_(rRCDD!wi#i8oAm&jYtn2_@VB%2-H*G%bN#|(6R6N?wM)3u`PiGzwuX7qmTgyF zpE)h0kuoxQ9?=kW7Y!=R@DmhU9)vwT*EZWzJ zrt+=2tqFts72yIp?|gvdLhs8Hfku^Z(){gmN%Y=K#P|%fkvgUj~HfIp3CuXqCtYGtJ#me+n+-LmP( z*XNuk%!aH8bIE@_Bj46>M*dSro|7<6vZ7WUHh5YQzN$>IJFqCb|CT!wj~R2C2%=q{ zpt8rzY$aw?W?=Ustv{jo?Ow@ZRkLe<)NItY>Cyhle*wR59dTdF6(@{5^ zAQBOB*hNtc3bkY-8{Cm$nFS@elbTtSqrt7MB{h_4y+~`!mVa}?c&N>&?P}GqdMuhQ z&@TD5Czd((DcG_Su~dKKV)Pj$-qi1WHM8_vc^O4?^!oY|tmK~i!{fjd&@_1E(T~r7 z_REZy&hMT^ySJB3W7l$4YhR`M(J7S5S~+4Q&3HPa)z%zPpisOp$^ zTEe99ig2$5_qFr!$;7A6CJ}PJmRhli>w?LC}Y`#HLGy6 zMU4EhL~dKCN5Ut;U2jd*83ShBNiu zcJB0l9>1Modc?-oM<R4?}3g}UJ%@K);kriq>)e*rh%hdqM)5Q)*+O8 zXm;SEbs@koiYS!9YXIclSg+5m_s~yrW#kKMdiRszg(gCP5HPmP7L)vCf8@fxUh6qY z@Z#TmkjzAZX{rwE+q|K~F2v5{_@vt%>yT_a#fF03SFt{0RXvDAiaY~K9CgS1O>frXgAjBCS}mEd4mIWZ$=ovd5| zR?GRdU}d6+Q`+JRW)|=v7$)XNkn3yE`!nAiSCvOB1jKT zG<1aK3s<0b0m==egTD#8i(Of=1pGDTOCho0XpIOMQ&P87cVKY1W=C6kIg z9cH=@a&zbm2+`|{(_?YC9fdm?1TY~-pwlBn?>=(~1pDKbco6jloP;0-cqRiwV1A_S zEyV0Dj8Pwy!nekzaN>{)7rgZ&_QLxK{~1yRe865^yx>}+a!ECd>#MMwddow z@CU{l+Rt$xuXuf}?ga{3IAr?Raql^c@a%sI0U5m}HvJ5O1#I%_MMPt#BH>OqUZ{-k zt>4Xzz=%jT*FVW(uYkWyx}9Gw$HdN*qU?Bit#ji(Wi7p-u|_8?h^%szIS^s^fNM}b zgGy>|=cbEufpguY5_6w~&ZLv=Bo06UF9EYIY;Er-1VK)SyF&!|J{axiE1z^(hXwVq zsFS=K-#zC}CcOs^8W{KAt+kK)jYDgDYbCXv{{rwsgqtIU3<910$CJi)s?? z_t8k{>7*0~4l~LLF7$WXT5OSq5QCTbP_l!SN|{R}3D&eWA8~0ltWh1IL+ZBX4rRSt zWF6Om3WDMu4xK^1(BF`2cL}rUCzhHAB`@j5&R-yk_l*t;mPGY|u2^o|myvcOdrg0W z%=lX;f^Vkqfp?u7*4qQq%A3Mpf!xspWBSKS@O%r*TSM}?dl(@*%{0Jm_8;(h{R__M Bt + images/ajax-loader-big.gif images/page-prev.gif images/page-next.gif images/0p.png diff --git a/bacula/src/qt-console/medialist/mediaview.cpp b/bacula/src/qt-console/medialist/mediaview.cpp index 8636e8369f..9aa1db348f 100644 --- a/bacula/src/qt-console/medialist/mediaview.cpp +++ b/bacula/src/qt-console/medialist/mediaview.cpp @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -107,7 +107,7 @@ void MediaView::purgePushed() bool MediaView::getSelection(QStringList &list) { QTableWidgetItem *it; - QList items = m_tableMedia->selectedItems(); + QList items = m_tableMedia->selectedItems(); int row; int nrows; /* number of rows */ bool *tab; @@ -121,7 +121,7 @@ bool MediaView::getSelection(QStringList &list) for (int i = 0; i < nb; ++i) { row = items[i]->row(); if (!tab[row]) { - tab[row] = true; + tab[row] = true; it = m_tableMedia->item(row, 0); list.append(it->text()); } diff --git a/bacula/src/qt-console/tray-monitor/authenticate.cpp b/bacula/src/qt-console/tray-monitor/authenticate.cpp index 522c323da6..339655c36d 100644 --- a/bacula/src/qt-console/tray-monitor/authenticate.cpp +++ b/bacula/src/qt-console/tray-monitor/authenticate.cpp @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -29,164 +29,117 @@ * */ -//#include "winhdrs.h" #include "tray-monitor.h" -void senditf(const char *fmt, ...); -void sendit(const char *buf); - /* Commands sent to Director */ static char DIRhello[] = "Hello %s calling\n"; +static char SDhello[] = "Hello SD: Bacula Director %s calling\n"; + /* Response from Director */ static char DIROKhello[] = "1000 OK:"; -/* Commands sent to Storage daemon and File daemon and received +/* Commands sent to File daemon and received * from the User Agent */ -static char SDFDhello[] = "Hello Director %s calling\n"; +static char FDhello[] = "Hello Director %s calling\n"; /* Response from SD */ -static char SDOKhello[] = "3000 OK Hello\n"; +static char SDOKhello[] = "3000 OK Hello"; /* Response from FD */ static char FDOKhello[] = "2000 OK Hello"; /* Forward referenced functions */ -/* - * Authenticate Director - */ -int authenticate_director(JCR *jcr, MONITOR *mon, DIRRES */*director*/) +int authenticate_daemon(JCR *jcr, MONITOR *mon, RESMON *res) { - BSOCK *dir = jcr->dir_bsock; + BSOCK *bs = res->bs; int tls_local_need = BNET_TLS_NONE; int tls_remote_need = BNET_TLS_NONE; int compatible = true; char bashed_name[MAX_NAME_LENGTH]; - char *password; + char *password, *p; + int ret = 0; bstrncpy(bashed_name, mon->hdr.name, sizeof(bashed_name)); bash_spaces(bashed_name); - password = mon->password; - - /* Timeout Hello after 5 mins */ - btimer_t *tid = start_bsock_timer(dir, 60 * 5); - dir->fsend(DIRhello, bashed_name); + password = res->password; - if (!cram_md5_respond(dir, password, &tls_remote_need, &compatible) || - !cram_md5_challenge(dir, password, tls_local_need, compatible)) { - stop_bsock_timer(tid); - Jmsg0(jcr, M_FATAL, 0, _("Director authorization problem.\n" - "Most likely the passwords do not agree.\n" - "For help, please see " MANUAL_AUTH_URL "\n")); - return 0; + /* TLS Requirement */ + if (res->tls_enable) { + tls_local_need = BNET_TLS_REQUIRED; } - Dmsg1(6, ">dird: %s", dir->msg); - if (dir->recv() <= 0) { - stop_bsock_timer(tid); - Jmsg1(jcr, M_FATAL, 0, _("Bad response to Hello command: ERR=%s\n"), - dir->bstrerror()); - return 0; - } - Dmsg1(10, "msg); - stop_bsock_timer(tid); - if (strncmp(dir->msg, DIROKhello, sizeof(DIROKhello)-1) != 0) { - Jmsg0(jcr, M_FATAL, 0, _("Director rejected Hello command\n")); - return 0; + /* Timeout Hello after 5 mins */ + btimer_t *tid = start_bsock_timer(bs, 60 * 5); + if (res->type == R_DIRECTOR) { + p = DIRhello; + } else if (res->type == R_STORAGE) { + p = SDhello; } else { - Jmsg0(jcr, M_INFO, 0, dir->msg); + p = FDhello; } - return 1; -} -/* - * Authenticate Storage daemon connection - */ -int authenticate_storage_daemon(JCR *jcr, MONITOR *monitor, STORE* store) -{ - BSOCK *sd = jcr->store_bsock; - char dirname[MAX_NAME_LENGTH]; - int tls_local_need = BNET_TLS_NONE; - int tls_remote_need = BNET_TLS_NONE; - int compatible = true; + bs->fsend(p, bashed_name); - /* - * Send my name to the Storage daemon then do authentication - */ - bstrncpy(dirname, monitor->hdr.name, sizeof(dirname)); - bash_spaces(dirname); - /* Timeout Hello after 5 mins */ - btimer_t *tid = start_bsock_timer(sd, 60 * 5); - if (!sd->fsend(SDFDhello, dirname)) { - stop_bsock_timer(tid); - Jmsg(jcr, M_FATAL, 0, _("Error sending Hello to Storage daemon. ERR=%s\n"), sd->bstrerror()); - return 0; + if (!cram_md5_respond(bs, password, &tls_remote_need, &compatible) || + !cram_md5_challenge(bs, password, tls_local_need, compatible)) { + Jmsg(jcr, M_FATAL, 0, _("Authorization problem.\n" + "Most likely the passwords do not agree.\n" + "For help, please see " MANUAL_AUTH_URL "\n")); + goto bail_out; } - if (!cram_md5_respond(sd, store->password, &tls_remote_need, &compatible) || - !cram_md5_challenge(sd, store->password, tls_local_need, compatible)) { - stop_bsock_timer(tid); - Jmsg0(jcr, M_FATAL, 0, _("Director and Storage daemon passwords or names not the same.\n" - "For help, please see " MANUAL_AUTH_URL "\n")); - return 0; - } - Dmsg1(116, ">stored: %s", sd->msg); - if (sd->recv() <= 0) { - stop_bsock_timer(tid); - Jmsg1(jcr, M_FATAL, 0, _("bdirdbstrerror()); - return 0; + + /* Verify that the remote host is willing to meet our TLS requirements */ + if (tls_remote_need < tls_local_need && tls_local_need != BNET_TLS_OK && tls_remote_need != BNET_TLS_OK) { + Jmsg(jcr, M_FATAL, 0, _("Authorization problem:" + " Remote server did not advertise required TLS support.\n")); + goto bail_out; } - Dmsg1(110, "msg); - stop_bsock_timer(tid); - if (strncmp(sd->msg, SDOKhello, sizeof(SDOKhello)) != 0) { - Jmsg0(jcr, M_FATAL, 0, _("Storage daemon rejected Hello command\n")); - return 0; + + /* Verify that we are willing to meet the remote host's requirements */ + if (tls_remote_need > tls_local_need && tls_local_need != BNET_TLS_OK && tls_remote_need != BNET_TLS_OK) { + Jmsg(jcr, M_FATAL, 0, ("Authorization problem:" + " Remote server requires TLS.\n")); + goto bail_out; } - return 1; -} -/* - * Authenticate File daemon connection - */ -int authenticate_file_daemon(JCR *jcr, MONITOR *monitor, CLIENT* client) -{ - BSOCK *fd = jcr->file_bsock; - char dirname[MAX_NAME_LENGTH]; - int tls_local_need = BNET_TLS_NONE; - int tls_remote_need = BNET_TLS_NONE; - int compatible = true; + /* Is TLS Enabled? */ + if (tls_local_need >= BNET_TLS_OK && tls_remote_need >= BNET_TLS_OK) { + /* Engage TLS! Full Speed Ahead! */ + if (!bnet_tls_client(res->tls_ctx, bs, NULL)) { + Jmsg(jcr, M_FATAL, 0, _("TLS negotiation failed\n")); + goto bail_out; + } + } - /* - * Send my name to the File daemon then do authentication - */ - bstrncpy(dirname, monitor->hdr.name, sizeof(dirname)); - bash_spaces(dirname); - /* Timeout Hello after 5 mins */ - btimer_t *tid = start_bsock_timer(fd, 60 * 5); - if (!fd->fsend(SDFDhello, dirname)) { - stop_bsock_timer(tid); - Jmsg(jcr, M_FATAL, 0, _("Error sending Hello to File daemon. ERR=%s\n"), fd->bstrerror()); - return 0; + Dmsg1(6, "> %s", bs->msg); + if (bs->recv() <= 0) { + Jmsg1(jcr, M_FATAL, 0, _("Bad response to Hello command: ERR=%s\n"), + bs->bstrerror()); + goto bail_out; } - if (!cram_md5_respond(fd, client->password, &tls_remote_need, &compatible) || - !cram_md5_challenge(fd, client->password, tls_local_need, compatible)) { - stop_bsock_timer(tid); - Jmsg(jcr, M_FATAL, 0, _("Director and File daemon passwords or names not the same.\n" - "For help, please see " MANUAL_AUTH_URL "\n")); - return 0; + Dmsg1(10, "< %s", bs->msg); + switch(res->type) { + case R_DIRECTOR: + p = DIROKhello; + break; + case R_CLIENT: + p = FDOKhello; + break; + case R_STORAGE: + p = SDOKhello; + break; } - Dmsg1(116, ">filed: %s", fd->msg); - if (fd->recv() <= 0) { - stop_bsock_timer(tid); - Jmsg(jcr, M_FATAL, 0, _("Bad response from File daemon to Hello command: ERR=%s\n"), - fd->bstrerror()); - return 0; + if (strncmp(bs->msg, p, strlen(p)) != 0) { + Jmsg(jcr, M_FATAL, 0, _("Daemon rejected Hello command\n")); + goto bail_out; + } else { + //Jmsg0(jcr, M_INFO, 0, dir->msg); } - Dmsg1(110, "msg); - stop_bsock_timer(tid); - if (strncmp(fd->msg, FDOKhello, sizeof(FDOKhello)-1) != 0) { - Jmsg(jcr, M_FATAL, 0, _("File daemon rejected Hello command\n")); - return 0; + ret = 1; +bail_out: + if (tid) { + stop_bsock_timer(tid); } - return 1; + return ret; } diff --git a/bacula/src/qt-console/tray-monitor/common.h b/bacula/src/qt-console/tray-monitor/common.h new file mode 100644 index 0000000000..ee555330cb --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/common.h @@ -0,0 +1,67 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef TRAY_MONITOR_COMMON_H +#define TRAY_MONITOR_COMMON_H + +#if defined(HAVE_WIN32) +#if !defined(_STAT_H) +#define _STAT_H /* don't pull in MinGW stat.h */ +#endif +#ifndef _STAT_DEFINED +#define _STAT_DEFINED /* don't pull in MinGW stat.h */ +#endif +#endif + +#if defined(HAVE_WIN32) +#if defined(HAVE_MINGW) +#include "mingwconfig.h" +#else +#include "winconfig.h" +#endif +#else +#include "config.h" +#endif +#define __CONFIG_H + + +#include +#include + +#include "bacula.h" + +class DbgLine +{ +public: + const char *funct; + DbgLine(const char *fun): funct(fun) { + Dmsg2(0, "[%p] -> %s\n", bthread_get_thread_id(), funct); + } + ~DbgLine() { + Dmsg2(0, "[%p] <- %s\n", bthread_get_thread_id(), funct); + } +}; + +#ifdef Enter +#undef Enter +#endif +//#define Enter() DbgLine _enter(__PRETTY_FUNCTION__) +#define Enter() + +#endif diff --git a/bacula/src/qt-console/tray-monitor/conf.cpp b/bacula/src/qt-console/tray-monitor/conf.cpp new file mode 100644 index 0000000000..729b559e79 --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/conf.cpp @@ -0,0 +1,412 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "conf.h" +#include "tray-monitor.h" +#include + +extern char *configfile; // defined in tray-monitor.cpp +extern RES_TABLE resources[]; +extern int32_t r_first; +extern int32_t r_last; +extern int32_t res_all_size; +extern URES res_all; + +bool Conf::parse_config() +{ + bool ret; + config = New(CONFIG()); + config->encode_password(false); + config->init(configfile, NULL, M_ERROR, (void *)&res_all, res_all_size, + r_first, r_last, resources, &rhead); + ret = config->parse_config(); + return ret; +} + +/* Check for \ at the end */ +static char *is_str_valid(POOLMEM **buf, const char *p) +{ + char *p1; + if (!p || !*p) { + return NULL; + } + p1 = *buf = check_pool_memory_size(*buf, (strlen(p) + 1)); + for (; *p ; p++) { + if (*p == '\\') { + *p1++ = '/'; + + } else if (*p == '"') { + return NULL; + + } else { + *p1++ = *p; + } + } + *p1 = 0; + return *buf; +} + +/* The .toUtf8().data() function can be called only one time at a time */ +void Conf::accept() +{ + POOLMEM *buf = get_pool_memory(PM_FNAME); + POOL_MEM tmp, tmp2, name; + QString str; + const char *restype=NULL, *p=NULL, *pname; + struct stat sp; + FILE *fp=NULL; + int n; + bool commit=false; + bool doclose=true; + bool ok; + + Mmsg(tmp, "%s.temp", configfile); + fp = fopen(tmp.c_str(), "w"); + if (!fp) { + berrno be; + display_error("Unable to open %s to write the new configuration file. ERR=%s\n", tmp.c_str(), be.bstrerror()); + goto bail_out; + } + str = UIConf.editName->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (!p) { + display_error(_("The Name of the Monitor should be set")); + doclose = false; + goto bail_out; + } + + fprintf(fp, "Monitor {\n Name=\"%s\"\n", p); + + n = UIConf.spinRefresh->value(); + fprintf(fp, " Refresh Interval = %d\n", n); + + if (UIConf.cbDspAdvanced->isChecked()) { + fprintf(fp, " Display Advanced Options = yes\n"); + } + + str = UIConf.editCommandDir->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (p) { // TODO: Check for \ at the end of the string + fprintf(fp, " Command Directory = \"%s\"\n", p); + } + + fprintf(fp, "}\n"); + + for (int i = 1; i < UIConf.tabWidget->count() ; i++) { + ConfTab *t = (ConfTab *) UIConf.tabWidget->widget(i); + if (t->isEnabled() == false) { + continue; // This one was deleted + } + for(int i = 0; resources[i].name ; i++) { + if (resources[i].rcode == t->res->type) { + restype = resources[i].name; + break; + } + } + if (!restype) { + goto bail_out; + } + + str = t->ui.editName->text(); + pname = is_str_valid(&buf, str.toUtf8().data()); + if (!pname) { + display_error(_("The name of the Resource should be set")); + doclose = false; + goto bail_out; + } + pm_strcpy(name, pname); + + str = t->ui.editAddress->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (!p) { + display_error(_("The address of the Resource should be set for resource %s"), name.c_str()); + doclose = false; + goto bail_out; + } + fprintf(fp, "%s {\n Name = \"%s\"\n Address = \"%s\"\n", restype, name.c_str(), p); + + str = t->ui.editPassword->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (!p) { + display_error(_("The Password of should be set for resource %s"), name.c_str()); + doclose = false; + goto bail_out; + } + fprintf(fp, " Password = \"%s\"\n", p); + + str = t->ui.editDescription->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (p) { + fprintf(fp, " Description = \"%s\"\n", p); + } + n = t->ui.editPort->text().toInt(&ok, 10); + if (ok && n > 0 && n < 65636) { + fprintf(fp, " Port = %d\n", n); + } + n = t->ui.editTimeout->text().toInt(&ok, 10); + if (ok && n > 0) { + fprintf(fp, " Connect Timeout = %d\n", n); + } + + str = t->ui.editCaCertificateFile->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (p) { + if (stat(p, &sp) != 0 || !S_ISREG(sp.st_mode)) { + display_error(_("The TLS CA Certificate File should be a PEM file for resource %s"), name.c_str()); + doclose = false; + goto bail_out; + } + fprintf(fp, " TLSCaCertificateFile = \"%s\"\n", p); + } + + str = t->ui.editCaCertificateDir->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (p) { + if (stat(p, &sp) != 0 || !S_ISDIR(sp.st_mode)) { + display_error(_("The TLS CA Certificate Directory should be a directory for resource %s"), name.c_str()); + doclose = false; + goto bail_out; + } + fprintf(fp, " TLSCaCertificateDir = \"%s\"\n", p); + } + + str = t->ui.editCertificate->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (p) { + if (stat(p, &sp) != 0 || !S_ISREG(sp.st_mode)) { + display_error(_("The TLS Certificate File should be a file for resource %s"), name.c_str()); + doclose = false; + goto bail_out; + } + fprintf(fp, " TLSCertificate = \"%s\"\n", p); + } + + str = t->ui.editKey->text(); + p = is_str_valid(&buf, str.toUtf8().data()); + if (p) { + if (stat(p, &sp) != 0 || !S_ISREG(sp.st_mode)) { + display_error(_("The TLS Key File should be a file for resource %s"), name.c_str()); + doclose = false; + goto bail_out; + } + fprintf(fp, " TLSKey = \"%s\"\n", p); + } + if (t->ui.cbTLSEnabled->isChecked()) { + fprintf(fp, " TLS Enable = yes\n"); + } + if (strcmp(restype, "client") == 0 && t->ui.cbRemote->isChecked()) { + fprintf(fp, " Remote = yes\n"); + } + if (strcmp(restype, "director") == 0 && t->ui.cbUseSetIp->isChecked()) { + fprintf(fp, " UseSetIp = yes\n"); + } + if (t->ui.cbMonitor->isChecked()) { + fprintf(fp, " Monitor = yes\n"); + } + fprintf(fp, "}\n"); + } + commit = true; + // Save the configuration file +bail_out: + if (fp) { + fclose(fp); + } + if (commit) { + // TODO: We probably need to load the configuration file to see if it works + unlink(configfile); + if (rename(tmp.c_str(), configfile) == 0) { + reload(); + + } else { + berrno be; + display_error("Unable to write to the configuration file %s ERR=%s\n", configfile, be.bstrerror()); + } + } + if (doclose) { + close(); + deleteLater(); + } +} + +Conf::~Conf() +{ + if (config) { + delete config; + } +} + +Conf::Conf(): QDialog() +{ + RESMON *res; + MONITOR *mon; + + rhead = NULL; + items = 0; + UIConf.setupUi(this); + if (parse_config()) { + for(res=NULL; (res=(RESMON *)GetNextRes(rhead, R_CLIENT, (RES*)res));) { + addResource(res, res->hdr.name); + } + for(res=NULL; (res=(RESMON *)GetNextRes(rhead, R_DIRECTOR, (RES*)res));) { + addResource(res, res->hdr.name); + } + for(res=NULL; (res=(RESMON *)GetNextRes(rhead, R_STORAGE, (RES*)res));) { + addResource(res, res->hdr.name); + } + mon = (MONITOR *)GetNextRes(rhead, R_MONITOR, NULL); + UIConf.editName->setText(QString(mon->hdr.name)); + UIConf.spinRefresh->setValue(mon->RefreshInterval); + UIConf.editCommandDir->setText(QString(NPRTB(mon->command_dir))); + + if (mon->display_advanced_options) { + UIConf.cbDspAdvanced->setChecked(true); + } + } + setAttribute(Qt::WA_DeleteOnClose, true); + show(); +} + +void Conf::addResource(RESMON *res, const char *title) +{ + char ed1[50]; + ConfTab *w = new ConfTab(res); + w->ui.editName->setText(QString(res->hdr.name)); + if (res->password) { + w->ui.editPassword->setText(QString(res->password)); + } + if (res->type != R_CLIENT) { + w->ui.cbRemote->hide(); + w->ui.labelRemote->hide(); + + } else if (res->use_remote) { + w->ui.cbRemote->setChecked(true); + } + + if (res->type != R_DIRECTOR) { + w->ui.cbUseSetIp->hide(); + w->ui.labelSetIp->hide(); + + } else if (res->use_setip) { + w->ui.cbUseSetIp->setChecked(true); + } + + if (res->use_monitor) { + w->ui.cbMonitor->setChecked(true); + } + w->ui.editAddress->setText(QString(res->address)); + w->ui.editPort->setText(QString(edit_uint64(res->port, ed1))); + w->ui.editTimeout->setText(QString(edit_uint64(res->connect_timeout, ed1))); + if (!res->tls_enable) { + if (w->ui.cbTLSEnabled->isChecked()) { + emit w->ui.cbTLSEnabled->click(); + } + } + if (res->tls_ca_certfile) { + w->ui.editCaCertificateFile->setText(QString(res->tls_ca_certfile)); + } + if (res->tls_ca_certdir) { + w->ui.editCaCertificateDir->setText(QString(res->tls_ca_certdir)); + } + if (res->tls_certfile) { + w->ui.editCertificate->setText(QString(res->tls_certfile)); + } + if (res->tls_keyfile) { + w->ui.editKey->setText(QString(res->tls_keyfile)); + } + + UIConf.tabWidget->addTab(w, QString(title)); + items++; +} + +void Conf::addRes(int type, const char *title) +{ + RESMON *res = (RESMON *) malloc(sizeof(RESMON)); + init_resource(config, type, res); + res->type = type; // Not sure it's set by init_resource + res->new_resource = true; // We want to free this resource with the ConfTab + addResource(res, title); +} + +void Conf::addDir() +{ + addRes(R_DIRECTOR, "New Director"); +} + +void Conf::addStore() +{ + addRes(R_STORAGE, "New Storage"); +} + +void Conf::addClient() +{ + addRes(R_CLIENT, "New Client"); +} + +void Conf::togglePassword() +{ + if (passtype == QLineEdit::Normal) { + passtype = QLineEdit::PasswordEchoOnEdit; + } else { + passtype = QLineEdit::Normal; + } + for (int i = 1; i < UIConf.tabWidget->count() ; i++) { + ConfTab *tab = (ConfTab *) UIConf.tabWidget->widget(i); + tab->ui.editPassword->setEchoMode(passtype); + } +} + +void Conf::selectCommandDir() +{ + QString directory = QFileDialog::getExistingDirectory(this, + tr("Select Command Directory"), + QDir::currentPath()); + UIConf.editCommandDir->setText(directory); +} + +void ConfTab::selectCaCertificateFile() +{ + QString directory = QFileDialog::getOpenFileName(this, + tr("Select CA Certificate File PEM file"), + QDir::currentPath()); + ui.editCaCertificateFile->setText(directory); +} + +void ConfTab::selectCaCertificateDir() +{ + QString directory = QFileDialog::getExistingDirectory(this, + tr("Select CA Certificate Directory"), + QDir::currentPath()); + ui.editCaCertificateDir->setText(directory); +} + +void ConfTab::selectCertificate() +{ + QString file = QFileDialog::getOpenFileName(this, + tr("Select TLS Certificate File"), + QDir::currentPath()); + ui.editCertificate->setText(file); +} + +void ConfTab::selectKey() +{ + QString file = QFileDialog::getOpenFileName(this, + tr("Select TLS Certificate File"), + QDir::currentPath()); + ui.editKey->setText(file); +} diff --git a/bacula/src/qt-console/tray-monitor/dir-monitor.ui b/bacula/src/qt-console/tray-monitor/dir-monitor.ui new file mode 100644 index 0000000000..0c68374fb9 --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/dir-monitor.ui @@ -0,0 +1,176 @@ + + + dirStatus + + + + 0 + 0 + 518 + 642 + + + + Form + + + + + + + + Director Status + + + + + + + + + + + + + Name: + + + + + + + Started: + + + + + + + + + + + + + + + + + + + + + Version: + + + + + + + Plugins: + + + + + + + + + + + + + + Reloaded: + + + + + + + + + + + + + + + + + Running Jobs + + + + + + QAbstractItemView::SingleSelection + + + false + + + + + + + + + + Terminated Jobs + + + + + + QAbstractItemView::SingleSelection + + + + + + + + + + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + :/images/view-refresh.png:/images/view-refresh.png + + + + + + + + + + + + diff --git a/bacula/src/qt-console/tray-monitor/dirstatus.cpp b/bacula/src/qt-console/tray-monitor/dirstatus.cpp new file mode 100644 index 0000000000..0e692a31ce --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/dirstatus.cpp @@ -0,0 +1,127 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "dirstatus.h" +#include "../util/fmtwidgetitem.h" +#include "jcr.h" + +void DIRStatus::doUpdate() +{ + if (count == 0) { + count++; + task *t = new task(); + status.pushButton->setEnabled(false); + connect(t, SIGNAL(done(task *)), this, SLOT(taskDone(task *)), Qt::QueuedConnection); + t->init(res, TASK_STATUS); + res->wrk->queue(t); + status.statusBar->setText(QString("Trying to connect to Director...")); + Dmsg1(50, "doUpdate(%p)\n", res); + } +} + +void DIRStatus::taskDone(task *t) +{ + count--; + if (!t->status) { + status.statusBar->setText(QString(t->errmsg)); + + } else { + status.statusBar->clear(); + if (t->type == TASK_STATUS) { + char ed1[50]; + struct s_last_job *ljob; + struct s_running_job *rjob; + res->mutex->lock(); + status.labelName->setText(QString(res->name)); + status.labelVersion->setText(QString(res->version)); + status.labelStarted->setText(QString(res->started)); + status.labelReloaded->setText(QString(res->reloaded)); + status.labelPlugins->setText(QString(res->plugins)); + /* Clear the table first */ + Freeze(*status.tableRunning); + Freeze(*status.tableTerminated); + QStringList headerlistR = (QStringList() << tr("JobId") + << tr("Job") << tr("Level") << tr("Client") + << tr("Status") << tr("Storage") + << tr("Files") << tr("Bytes") << tr("Errors")); + status.tableRunning->clear(); + status.tableRunning->setRowCount(0); + status.tableRunning->setColumnCount(headerlistR.count()); + status.tableRunning->setHorizontalHeaderLabels(headerlistR); + status.tableRunning->setEditTriggers(QAbstractItemView::NoEditTriggers); + status.tableRunning->verticalHeader()->hide(); + status.tableRunning->setSortingEnabled(true); + + if (res->running_jobs) { + status.tableRunning->setRowCount(res->running_jobs->size()); + int row=0; + foreach_alist(rjob, res->running_jobs) { + int col=0; + TableItemFormatter item(*status.tableRunning, row++); + item.setNumericFld(col++, QString(edit_uint64(rjob->JobId, ed1))); + item.setTextFld(col++, QString(rjob->Job)); + item.setJobLevelFld(col++, QString(rjob->JobLevel)); + item.setTextFld(col++, QString(rjob->Client)); + item.setJobStatusFld(col++, QString(rjob->JobStatus)); + item.setTextFld(col++, QString(rjob->Storage)); + item.setNumericFld(col++, QString(edit_uint64(rjob->JobFiles, ed1))); + item.setBytesFld(col++, QString(edit_uint64(rjob->JobBytes, ed1))); + item.setNumericFld(col++, QString(edit_uint64(rjob->Errors, ed1))); + } + } else { + Dmsg0(0, "Strange, the list is NULL\n"); + } + + QStringList headerlistT = (QStringList() << tr("JobId") + << tr("Job") << tr("Level") + << tr("Status") << tr("Files") << tr("Bytes") + << tr("Errors")); + + status.tableTerminated->clear(); + status.tableTerminated->setRowCount(0); + status.tableTerminated->setColumnCount(headerlistT.count()); + status.tableTerminated->setHorizontalHeaderLabels(headerlistT); + status.tableTerminated->setEditTriggers(QAbstractItemView::NoEditTriggers); + status.tableTerminated->verticalHeader()->hide(); + status.tableTerminated->setSortingEnabled(true); + + if (res->terminated_jobs) { + status.tableTerminated->setRowCount(res->terminated_jobs->size()); + int row=0; + foreach_dlist(ljob, res->terminated_jobs) { + int col=0; + TableItemFormatter item(*status.tableTerminated, row++); + item.setNumericFld(col++, QString(edit_uint64(ljob->JobId, ed1))); + item.setTextFld(col++, QString(ljob->Job)); + item.setJobLevelFld(col++, QString(ljob->JobLevel)); + item.setJobStatusFld(col++, QString(ljob->JobStatus)); + item.setNumericFld(col++, QString(edit_uint64(ljob->JobFiles, ed1))); + item.setBytesFld(col++, QString(edit_uint64(ljob->JobBytes, ed1))); + item.setNumericFld(col++, QString(edit_uint64(ljob->Errors, ed1))); + } + } else { + Dmsg0(0, "Strange, the list is NULL\n"); + } + res->mutex->unlock(); + } + Dmsg1(50, " Task %p OK\n", t); + } + t->deleteLater(); + status.pushButton->setEnabled(true); +} diff --git a/bacula/src/qt-console/tray-monitor/dirstatus.h b/bacula/src/qt-console/tray-monitor/dirstatus.h new file mode 100644 index 0000000000..242bd69a76 --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/dirstatus.h @@ -0,0 +1,42 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "common.h" +#include "ui_dir-monitor.h" +#include "task.h" +#include "status.h" + +class DIRStatus: public ResStatus +{ + Q_OBJECT + +public: + Ui::dirStatus status; + + DIRStatus(RESMON *d): ResStatus(d) + { + status.setupUi(this); + QObject::connect(status.pushButton, SIGNAL(clicked()), this, SLOT(doUpdate()), Qt::QueuedConnection); + }; + ~DIRStatus() { + }; +public slots: + void doUpdate(); + void taskDone(task *); +}; diff --git a/bacula/src/qt-console/tray-monitor/fd-monitor.ui b/bacula/src/qt-console/tray-monitor/fd-monitor.ui new file mode 100644 index 0000000000..fee5f730c7 --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/fd-monitor.ui @@ -0,0 +1,176 @@ + + + fdStatus + + + + 0 + 0 + 518 + 435 + + + + Form + + + + + + + + FileDaemon Status + + + + + + + + + + + + + + + + + + + + Name: + + + + + + + Bandwidth Limit: + + + + + + + + + + + + + + Started: + + + + + + + + + + + + + + + + + + + + + Version: + + + + + + + Plugins: + + + + + + + + + + Running Jobs + + + + + + QAbstractItemView::SingleSelection + + + false + + + + + + + + + + Terminated Jobs + + + + + + QAbstractItemView::SingleSelection + + + + + + + + + + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + :/images/view-refresh.png:/images/view-refresh.png + + + + + + + + + + + + diff --git a/bacula/src/qt-console/tray-monitor/fdstatus.cpp b/bacula/src/qt-console/tray-monitor/fdstatus.cpp new file mode 100644 index 0000000000..a09a21479e --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/fdstatus.cpp @@ -0,0 +1,125 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "fdstatus.h" +#include "../util/fmtwidgetitem.h" +#include "jcr.h" + +void FDStatus::doUpdate() +{ + if (count == 0) { // We do only doUpdate() + count++; + task *t = new task(); + status.pushButton->setEnabled(false); + connect(t, SIGNAL(done(task *)), this, SLOT(taskDone(task *)), Qt::QueuedConnection); + t->init(res, TASK_STATUS); + res->wrk->queue(t); + status.statusBar->setText(QString("Trying to connect to FD...")); + Dmsg1(50, "doUpdate(%p)\n", res); + } +} + +void FDStatus::taskDone(task *t) +{ + int nbjobs=0; + count--; + if (!t->status) { + status.statusBar->setText(QString(t->errmsg)); + + } else if (isVisible()) { + status.statusBar->clear(); + if (t->type == TASK_STATUS) { + char ed1[50]; + struct s_last_job *ljob; + struct s_running_job *rjob; + res->mutex->lock(); + status.labelName->setText(QString(res->name)); + status.labelVersion->setText(QString(res->version)); + status.labelPlugins->setText(QString(res->plugins)); + status.labelStarted->setText(QString(res->started)); + status.labelBandwidth->setText(QString(edit_uint64(res->bwlimit, ed1))); + /* Clear the table first */ + Freeze(*status.tableRunning); + Freeze(*status.tableTerminated); + QStringList headerlistR = (QStringList() << tr("JobId") + << tr("Job") << tr("Level") + << tr("Files") << tr("Bytes") << tr("Errors") << tr("Current File")); + status.tableRunning->clear(); + status.tableRunning->setRowCount(0); + status.tableRunning->setColumnCount(headerlistR.count()); + status.tableRunning->setHorizontalHeaderLabels(headerlistR); + status.tableRunning->setEditTriggers(QAbstractItemView::NoEditTriggers); + status.tableRunning->verticalHeader()->hide(); + status.tableRunning->setSortingEnabled(true); + + if (res->running_jobs) { + status.tableRunning->setRowCount(res->running_jobs->size()); + int row=0; + foreach_alist(rjob, res->running_jobs) { + int col=0; + TableItemFormatter item(*status.tableRunning, row++); + item.setNumericFld(col++, QString(edit_uint64(rjob->JobId, ed1))); + item.setTextFld(col++, QString(rjob->Job)); + item.setJobLevelFld(col++, QString(rjob->JobLevel)); + item.setNumericFld(col++, QString(edit_uint64(rjob->JobFiles, ed1))); + item.setBytesFld(col++, QString(edit_uint64(rjob->JobBytes, ed1))); + item.setNumericFld(col++, QString(edit_uint64(rjob->Errors, ed1))); + item.setNumericFld(col++, QString(rjob->CurrentFile)); + nbjobs++; + } + } else { + Dmsg0(0, "Strange, the list is NULL\n"); + } + + QStringList headerlistT = (QStringList() << tr("JobId") + << tr("Job") << tr("Level") + << tr("Status") << tr("Files") << tr("Bytes") << tr("Errors")); + + status.tableTerminated->clear(); + status.tableTerminated->setRowCount(0); + status.tableTerminated->setColumnCount(headerlistT.count()); + status.tableTerminated->setHorizontalHeaderLabels(headerlistT); + status.tableTerminated->setEditTriggers(QAbstractItemView::NoEditTriggers); + status.tableTerminated->verticalHeader()->hide(); + status.tableTerminated->setSortingEnabled(true); + + if (res->terminated_jobs) { + status.tableTerminated->setRowCount(res->terminated_jobs->size()); + int row=0; + foreach_dlist(ljob, res->terminated_jobs) { + int col=0; + TableItemFormatter item(*status.tableTerminated, row++); + item.setNumericFld(col++, QString(edit_uint64(ljob->JobId, ed1))); + item.setTextFld(col++, QString(ljob->Job)); + item.setJobLevelFld(col++, QString(ljob->JobLevel)); + item.setJobStatusFld(col++, QString(ljob->JobStatus)); + item.setNumericFld(col++, QString(edit_uint64(ljob->JobFiles, ed1))); + item.setBytesFld(col++, QString(edit_uint64(ljob->JobBytes, ed1))); + item.setNumericFld(col++, QString(edit_uint64(ljob->Errors, ed1))); + } + } else { + Dmsg0(0, "Strange, the list is NULL\n"); + } + res->mutex->unlock(); + } + Dmsg1(50, " Task %p OK\n", t); + } + t->deleteLater(); + status.pushButton->setEnabled(true); +} diff --git a/bacula/src/qt-console/tray-monitor/fdstatus.h b/bacula/src/qt-console/tray-monitor/fdstatus.h new file mode 100644 index 0000000000..a3316d1368 --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/fdstatus.h @@ -0,0 +1,42 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "common.h" +#include "ui_fd-monitor.h" +#include "task.h" +#include "status.h" + +class FDStatus: public ResStatus +{ + Q_OBJECT + +public: + Ui::fdStatus status; + + FDStatus(RESMON *c): ResStatus(c) + { + status.setupUi(this); + QObject::connect(status.pushButton, SIGNAL(clicked()), this, SLOT(doUpdate()), Qt::QueuedConnection); + }; + ~FDStatus() { + }; +public slots: + void doUpdate(); + void taskDone(task *); +}; diff --git a/bacula/src/qt-console/tray-monitor/install_conf_file.in b/bacula/src/qt-console/tray-monitor/install_conf_file.in new file mode 100755 index 0000000000..6b9d27823b --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/install_conf_file.in @@ -0,0 +1,16 @@ +#!/bin/sh + +sbindir=@sbindir@ +sysconfdir=@sysconfdir@ +INSTALL_CONFIG="@INSTALL@ -m 640" +DESTDIR=`echo ${DESTDIR}` + +srcconf=bacula-tray-monitor.conf +if test -f ${DESTDIR}${sysconfdir}/${srcconf}; then + destconf=${srcconf}.new + echo " ==> Found existing $srcconf, installing new conf file as ${destconf}" +else + destconf=${srcconf} +fi +echo "${INSTALL_CONFIG} ${srcconf} ${DESTDIR}${sysconfdir}/${destconf}" +${INSTALL_CONFIG} ${srcconf} ${DESTDIR}${sysconfdir}/${destconf} diff --git a/bacula/src/qt-console/tray-monitor/task.h b/bacula/src/qt-console/tray-monitor/task.h new file mode 100644 index 0000000000..0f529a3c3f --- /dev/null +++ b/bacula/src/qt-console/tray-monitor/task.h @@ -0,0 +1,115 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef TASK_H +#define TASK_H + +#include "common.h" +#include +#include "tray_conf.h" + +enum { + TASK_NONE, + TASK_STATUS, + TASK_RESOURCES, + TASK_QUERY, + TASK_RUN, + TASK_RESTORE, + TASK_DEFAULTS, + TASK_CLOSE, + TASK_INFO, + TASK_BWLIMIT, + TASK_DISCONNECT +}; + +/* The task should emit a signal when done */ +class task: public QObject +{ + Q_OBJECT + +public: + RESMON *res; + POOLMEM *errmsg; + int type; + bool status; + char *curline; + char *curend; + char *arg; /* Argument that can be used by some tasks */ + char *arg2; + + union { + bool b; + int i; + char c[256]; + } result; /* The task might return something */ + + task(): QObject(), res(NULL), type(TASK_NONE), status(false), curline(NULL), + curend(NULL), arg(NULL), arg2(NULL) + { + errmsg = get_pool_memory(PM_FNAME); + *errmsg = 0; + memset(result.c, 0, sizeof(result.c)); + }; + ~task() { + Enter(); + disconnect(); /* disconnect all signals */ + free_pool_memory(errmsg); + }; + void init(int t) { + res = NULL; + type = t; + status = false; + }; + void init(RESMON *s, int t) { + init(t); + res = s; + }; + + RESMON *get_res(); + void lock_res(); + void unlock_res(); + bool connect_bacula(); + bool do_status(); + bool read_status_terminated(RESMON *res); + bool read_status_header(RESMON *res); + bool read_status_running(RESMON *res); + bool set_bandwidth(); + bool disconnect_bacula(); + void mark_as_done() { + status = true; + emit done(this); + }; + void mark_as_failed() { + status = false; + emit done(this); + }; + bool get_resources(); + bool get_next_line(RESMON *res); + bool get_job_defaults(); /* Look r->defaults.job */ + bool run_job(); + bool get_job_info(const char *level); /* look r->info */ + +signals: + void done(task *t); +}; + +worker *worker_start(); +void worker_stop(worker *); + +#endif diff --git a/bacula/src/qt-console/tray-monitor/tray-monitor.cpp b/bacula/src/qt-console/tray-monitor/tray-monitor.cpp index d63314bf4b..c6c15c6b39 100644 --- a/bacula/src/qt-console/tray-monitor/tray-monitor.cpp +++ b/bacula/src/qt-console/tray-monitor/tray-monitor.cpp @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,114 +11,175 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ -#include "tray-ui.h" - -int doconnect(monitoritem* item); -int docmd(monitoritem* item, const char* command); -static int authenticate_daemon(monitoritem* item, JCR *jcr); -/* Imported functions */ -int authenticate_director(JCR *jcr, MONITOR *monitor, DIRRES *director); -int authenticate_file_daemon(JCR *jcr, MONITOR *monitor, CLIENT* client); -int authenticate_storage_daemon(JCR *jcr, MONITOR *monitor, STORE* store); -extern bool parse_tmon_config(CONFIG *config, const char *configfile, int exit_code); -void get_list(monitoritem* item, const char *cmd, QStringList &lst); - -/* Dummy functions */ -int generate_daemon_event(JCR *, const char *) { return 1; } +#include "tray-monitor.h" +#include +#include /* Static variables */ -static char *configfile = NULL; -static MONITOR *monitor; -static JCR jcr; -static int nitems = 0; -static monitoritem items[32]; -static CONFIG *config; -static TrayUI *tray; - -/* Data received from DIR/FD/SD */ -//static char OKqstatus[] = "%c000 OK .status\n"; -//static char DotStatusJob[] = "JobId=%d JobStatus=%c JobErrors=%d\n"; - - -void updateStatusIcon(monitoritem* item); -void changeStatusMessage(monitoritem* item, const char *fmt,...); - -#define CONFIG_FILE "./tray-monitor.conf" /* default configuration file */ +char *configfile = NULL; +static MONITOR *monitor = NULL; +static CONFIG *config = NULL; +static TrayUI *mainwidget = NULL; +static TSched *scheduler = NULL; + +#define CONFIG_FILE "./bacula-tray-monitor.conf" /* default configuration file */ + +#ifdef HAVE_WIN32 +#define HOME_VAR "APPDATA" +#define CONFIG_FILE_HOME "bacula-tray-monitor.conf" /* In $HOME */ +#else +#define HOME_VAR "HOME" +#define CONFIG_FILE_HOME ".bacula-tray-monitor.conf" /* In $HOME */ +#endif static void usage() { fprintf(stderr, _( PROG_COPYRIGHT -"\nVersion: %s (%s) %s %s %s\n\n" +"\n%sVersion: %s (%s) %s %s %s\n\n" "Usage: tray-monitor [-c config_file] [-d debug_level]\n" " -c set configuration file to file\n" " -d set debug level to \n" " -dt print timestamp in debug output\n" " -t test - read configuration and exit\n" +" -W 0/1 force the detection of the systray\n" " -? print this message.\n" -"\n"), 2004, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER); +"\n"), 2004, "", VERSION, BDATE, HOST_OS, DISTNAME, DISTVER); } -int sm_line = 0; +void refresh_tray(TrayUI *t) +{ + RESMON *r; + MONITOR *mon; + + if (!t) { + return; + } + + t->clearTabs(); + if (!config) { + return; + } + + mon = (MONITOR *) GetNextRes(R_MONITOR, NULL); + t->spinRefresh->setValue(mon?mon->RefreshInterval:60); + + foreach_res(r, R_CLIENT) { + t->addTab(r); + } + foreach_res(r, R_DIRECTOR) { + t->addTab(r); + } + foreach_res(r, R_STORAGE) { + t->addTab(r); + } +} -void dotest() +void display_error(const char *fmt, ...) { - for (int i = 0; i < nitems; i++) { - const char *cmd; - - switch (items[i].type) { - case R_DIRECTOR: - cmd = ".jobs type=B"; - tray->clearText(items[i].get_name()); - docmd(&items[i], cmd); - break; - default: - break; - } + va_list arg_ptr; + POOL_MEM tmp(PM_MESSAGE); + QMessageBox msgBox; + int maxlen; + + if (!fmt || !*fmt) { + return; } + + maxlen = tmp.size() - 1; + va_start(arg_ptr, fmt); + bvsnprintf(tmp.c_str(), maxlen, fmt, arg_ptr); + va_end(arg_ptr); + + msgBox.setIcon(QMessageBox::Critical); + msgBox.setText(tmp.c_str()); + msgBox.exec(); } -void get_list(monitoritem *item, const char *cmd, QStringList &lst) +void error_handler(const char *file, int line, LEX */* lc */, const char *msg, ...) { - int stat; - - doconnect(item); - item->writecmd(cmd); - while((stat = item->D_sock->recv()) >= 0) { - strip_trailing_junk(item->D_sock->msg); - if (*(item->D_sock->msg)) { - lst << QString(item->D_sock->msg); - } + POOL_MEM tmp; + va_list arg_ptr; + va_start(arg_ptr, msg); + vsnprintf(tmp.c_str(), tmp.size(), msg, arg_ptr); + va_end(arg_ptr); + display_error("Error %s:%d %s\n", file, line, tmp.c_str()); +} + +int tls_pem_callback(char *buf, int size, const void * /*userdata*/) +{ + bool ok; + QString text = QInputDialog::getText(mainwidget, _("TLS PassPhrase"), + buf, QLineEdit::Normal, + QDir::home().dirName(), &ok); + if (ok) { + bstrncpy(buf, text.toUtf8().data(), size); + return 1; + } else { + return 0; } } -void refresh_item() +bool reload() { - for (int i = 0; i < nitems; i++) { - const char *cmd; - tray->clearText(items[i].get_name()); - switch (items[i].type) { - case R_DIRECTOR: - cmd = "status dir"; - break; - case R_CLIENT: - cmd = "status"; - break; - case R_STORAGE: - cmd = "status"; - break; - default: - exit(1); - break; - } - docmd(&items[i], cmd); + bool displaycfg=false; + int nitems = 0; + struct stat sp; + + Dmsg0(50, "reload the configuration!\n"); + scheduler->stop(); + if (config) { + delete config; + } + config = NULL; + monitor = NULL; + + if (stat(configfile, &sp) != 0) { + berrno be; + Dmsg2(50, "Unable to find %s. ERR=%s\n", configfile, be.bstrerror()); + displaycfg = true; + goto bail_out; + } + + config = New(CONFIG()); + if (!parse_tmon_config(config, configfile, M_ERROR)) { + Dmsg1(50, "Error while parsing %s\n", configfile); + // TODO: Display a warning message an open the configuration + // window + displaycfg = true; + } + + LockRes(); + foreach_res(monitor, R_MONITOR) { + nitems++; + } + if (!displaycfg && nitems != 1) { + Mmsg(config->m_errmsg, + _("Error: %d Monitor resources defined in %s. " + "You must define one Monitor resource.\n"), + nitems, configfile); + displaycfg = true; + } + monitor = (MONITOR*)GetNextRes(R_MONITOR, (RES *)NULL); + UnlockRes(); + if (displaycfg) { + display_error(config->m_errmsg); } + refresh_tray(mainwidget); + if (monitor && monitor->command_dir) { + scheduler->init(monitor->command_dir); + scheduler->start(); + } else { + Dmsg0(50, "Do not start the scheduler\n"); + } +bail_out: + return displaycfg; } /********************************************************************* @@ -128,12 +189,12 @@ void refresh_item() */ int main(int argc, char *argv[]) { - int ch, i, dir_index=-1; - bool test_config = false; - DIRRES* dird; - CLIENT* filed; - STORE* stored; - + QApplication app(argc, argv); + int ch; + bool test_config = false, display_cfg = false; + TrayUI tray; + TSched sched; + setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); @@ -142,7 +203,13 @@ int main(int argc, char *argv[]) my_name_is(argc, argv, "tray-monitor"); lmgr_init_thread(); init_msg(NULL, NULL, NULL); - working_directory = "/tmp"; +#ifdef HAVE_WIN32 + working_directory = getenv("TMP"); +#endif + if (working_directory == NULL) { + working_directory = "/tmp"; + } + start_watchdog(); #ifndef HAVE_WIN32 struct sigaction sigignore; @@ -152,7 +219,7 @@ int main(int argc, char *argv[]) sigaction(SIGPIPE, &sigignore, NULL); #endif - while ((ch = getopt(argc, argv, "bc:d:th?f:s:")) != -1) { + while ((ch = getopt(argc, argv, "c:d:th?TW:")) != -1) { switch (ch) { case 'c': /* configuration file */ if (configfile != NULL) { @@ -161,6 +228,14 @@ int main(int argc, char *argv[]) configfile = bstrdup(optarg); break; + case 'W': + tray.have_systray = (atoi(optarg) != 0); + break; + + case 'T': + set_trace(true); + break; + case 'd': if (*optarg == 't') { dbg_timestamp = true; @@ -191,283 +266,55 @@ int main(int argc, char *argv[]) exit(1); } - if (configfile == NULL) { - configfile = bstrdup(CONFIG_FILE); - } + /* Keep generated files for ourself */ + umask(0077); - config = new_config_parser(); - parse_tmon_config(config, configfile, M_ERROR_TERM); + if (configfile == NULL) { + if (getenv(HOME_VAR) != NULL) { + int len = strlen(getenv(HOME_VAR)) + strlen(CONFIG_FILE_HOME) + 5; + configfile = (char *) malloc(len); + bsnprintf(configfile, len, "%s/%s", getenv(HOME_VAR), CONFIG_FILE_HOME); - LockRes(); - nitems = 0; - foreach_res(monitor, R_MONITOR) { - nitems++; + } else { + configfile = bstrdup(CONFIG_FILE); + } } + Dmsg1(50, "configfile=%s\n", configfile); - if (nitems != 1) { - Emsg2(M_ERROR_TERM, 0, - _("Error: %d Monitor resources defined in %s. You must define one and only one Monitor resource.\n"), nitems, configfile); - } + // We need to initialize the scheduler before the reload() command + scheduler = &sched; - nitems = 0; - foreach_res(dird, R_DIRECTOR) { - dir_index=nitems; - items[nitems].type = R_DIRECTOR; - items[nitems].resource = dird; - items[nitems].D_sock = NULL; - items[nitems].state = warn; - items[nitems].oldstate = warn; - nitems++; - } - foreach_res(filed, R_CLIENT) { - items[nitems].type = R_CLIENT; - items[nitems].resource = filed; - items[nitems].D_sock = NULL; - items[nitems].state = warn; - items[nitems].oldstate = warn; - nitems++; - } - foreach_res(stored, R_STORAGE) { - items[nitems].type = R_STORAGE; - items[nitems].resource = stored; - items[nitems].D_sock = NULL; - items[nitems].state = warn; - items[nitems].oldstate = warn; - nitems++; - } - UnlockRes(); + OSDependentInit(); /* Initialize Windows path handling */ + (void)WSA_Init(); /* Initialize Windows sockets */ - if (nitems == 0) { - Emsg1(M_ERROR_TERM, 0, _("No Client, Storage or Director resource defined in %s\n" -"Without that I don't how to get status from the File, Storage or Director Daemon :-(\n"), configfile); - } + display_cfg = reload(); if (test_config) { exit(0); } + /* If we have a systray, we always keep the application*/ + if (tray.have_systray) { + app.setQuitOnLastWindowClosed(false); - (void)WSA_Init(); /* Initialize Windows sockets */ - - LockRes(); - monitor = (MONITOR*)GetNextRes(R_MONITOR, (RES *)NULL); - UnlockRes(); - - if ((monitor->RefreshInterval < 1) || (monitor->RefreshInterval > 600)) { - Emsg2(M_ERROR_TERM, 0, _("Invalid refresh interval defined in %s\n" -"This value must be greater or equal to 1 second and less or equal to 10 minutes (read value: %d).\n"), configfile, monitor->RefreshInterval); - } - - sm_line = 0; - QApplication app(argc, argv); - app.setQuitOnLastWindowClosed(false); - tray = new TrayUI(); - tray->setupUi(tray); - tray->spinRefresh->setValue(monitor->RefreshInterval); - if (dir_index >= 0) { - tray->addDirector(&items[dir_index]); + } else { /* Without a systray, we quit when we close */ + app.setQuitOnLastWindowClosed(true); } - - for (i = 0; i < nitems; i++) { - const char *cmd; - tray->addTab(items[i].get_name()); - switch (items[i].type) { - case R_DIRECTOR: - tray->addDirector(&items[i]); - cmd = "status dir"; - break; - case R_CLIENT: - cmd = "status"; - break; - case R_STORAGE: - cmd = "status"; - break; - default: - exit(1); - break; - } - docmd(&items[i], cmd); + tray.setupUi(&tray, monitor); + refresh_tray(&tray); + mainwidget = &tray; + if (display_cfg) { + new Conf(); } - - tray->startTimer(); - app.exec(); + sched.stop(); + stop_watchdog(); + (void)WSACleanup(); /* Cleanup Windows sockets */ - for (i = 0; i < nitems; i++) { - if (items[i].D_sock) { - items[i].writecmd("quit"); - if (items[i].D_sock) { - items[i].D_sock->signal(BNET_TERMINATE); /* send EOF */ - free_bsock(items[i].D_sock); - } - } + if (config) { + delete config; } - - - (void)WSACleanup(); /* Cleanup Windows sockets */ - - config->free_resources(); - free(config); config = NULL; + bfree_and_null(configfile); term_msg(); return 0; } - -static int authenticate_daemon(monitoritem* item, JCR *jcr) { - switch (item->type) { - case R_DIRECTOR: - return authenticate_director(jcr, monitor, (DIRRES*)item->resource); - case R_CLIENT: - return authenticate_file_daemon(jcr, monitor, (CLIENT*)item->resource); - case R_STORAGE: - return authenticate_storage_daemon(jcr, monitor, (STORE*)item->resource); - default: - printf(_("Error, currentitem is not a Client or a Storage..\n")); - return FALSE; - } - return false; -} - -void changeStatusMessage(monitoritem*, const char *fmt,...) { - char buf[512]; - va_list arg_ptr; - - va_start(arg_ptr, fmt); - bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); - va_end(arg_ptr); - tray->statusbar->showMessage(QString(buf)); -} - -int doconnect(monitoritem* item) -{ - if (!is_bsock_open(item->D_sock)) { - memset(&jcr, 0, sizeof(jcr)); - - DIRRES* dird; - CLIENT* filed; - STORE* stored; - - switch (item->type) { - case R_DIRECTOR: - dird = (DIRRES*)item->resource; - changeStatusMessage(item, _("Connecting to Director %s:%d"), dird->address, dird->DIRport); - if (!item->D_sock) { - item->D_sock = new_bsock(); - } - item->D_sock->connect(NULL, monitor->DIRConnectTimeout, - 0, 0, _("Director daemon"), dird->address, NULL, dird->DIRport, 0); - jcr.dir_bsock = item->D_sock; - break; - case R_CLIENT: - filed = (CLIENT*)item->resource; - changeStatusMessage(item, _("Connecting to Client %s:%d"), filed->address, filed->FDport); - if (!item->D_sock) { - item->D_sock = new_bsock(); - } - item->D_sock->connect(NULL, monitor->FDConnectTimeout, - 0, 0, _("File daemon"), filed->address, NULL, filed->FDport, 0); - jcr.file_bsock = item->D_sock; - break; - case R_STORAGE: - stored = (STORE*)item->resource; - changeStatusMessage(item, _("Connecting to Storage %s:%d"), stored->address, stored->SDport); - if (!item->D_sock) { - item->D_sock = new_bsock(); - } - item->D_sock->connect(NULL, monitor->SDConnectTimeout, - 0, 0, _("Storage daemon"), stored->address, NULL, stored->SDport, 0); - jcr.store_bsock = item->D_sock; - break; - default: - printf(_("Error, currentitem is not a Client, a Storage or a Director..\n")); - return 0; - } - - if (item->D_sock == NULL) { - changeStatusMessage(item, _("Cannot connect to daemon.")); - item->state = error; - item->oldstate = error; - return 0; - } - - if (!authenticate_daemon(item, &jcr)) { - item->state = error; - item->oldstate = error; - changeStatusMessage(item, _("Authentication error : %s"), item->D_sock->msg); - item->D_sock = NULL; - return 0; - } - - switch (item->type) { - case R_DIRECTOR: - changeStatusMessage(item, _("Opened connection with Director daemon.")); - break; - case R_CLIENT: - changeStatusMessage(item, _("Opened connection with File daemon.")); - break; - case R_STORAGE: - changeStatusMessage(item, _("Opened connection with Storage daemon.")); - break; - default: - printf(_("Error, currentitem is not a Client, a Storage or a Director..\n")); - return 0; - break; - } - - if (item->type == R_DIRECTOR) { /* Read connection messages... */ - docmd(item, ""); /* Usually invalid, but no matter */ - } - } - return 1; -} - -int docmd(monitoritem* item, const char* command) -{ - int stat; - //qDebug() << "docmd(" << item->get_name() << "," << command << ")"; - if (!doconnect(item)) { - return 0; - } - - if (command[0] != 0) - item->writecmd(command); - - while(1) { - if ((stat = item->D_sock->recv()) >= 0) { - strip_trailing_newline(item->D_sock->msg); - tray->appendText(item->get_name(), item->D_sock->msg); - } - else if (stat == BNET_SIGNAL) { - if (item->D_sock->msglen == BNET_EOD) { - // qDebug() << "<< EOD >>"; - return 1; - } - else if (item->D_sock->msglen == BNET_SUB_PROMPT) { - // qDebug() << "<< PROMPT >>"; - return 0; - } - else if (item->D_sock->msglen == BNET_HEARTBEAT) { - item->D_sock->signal(BNET_HB_RESPONSE); - } - else { - qDebug() << bnet_sig_to_ascii(item->D_sock->msglen); - } - } - else { /* BNET_HARDEOF || BNET_ERROR */ - item->D_sock = NULL; - item->state = error; - item->oldstate = error; - changeStatusMessage(item, _("Error : BNET_HARDEOF or BNET_ERROR")); - //fprintf(stderr, _("<< ERROR >>\n")); - return 0; - } - - if (item->D_sock->is_stop()) { - item->D_sock = NULL; - item->state = error; - item->oldstate = error; - changeStatusMessage(item, _("Error : Connection closed.")); - //fprintf(stderr, "<< STOP >>\n"); - return 0; /* error or term */ - } - } -} diff --git a/bacula/src/qt-console/tray-monitor/tray-monitor.h b/bacula/src/qt-console/tray-monitor/tray-monitor.h index 762c818928..8bf2225572 100644 --- a/bacula/src/qt-console/tray-monitor/tray-monitor.h +++ b/bacula/src/qt-console/tray-monitor/tray-monitor.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,168 +11,21 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ -/* - * Includes specific to the tray monitor - * - * Nicolas Boichat, August MMIV - * - */ #ifndef TRAY_MONITOR_H #define TRAY_MONITOR_H -#ifdef HAVE_WIN32 -# ifndef _STAT_DEFINED -# define _STAT_DEFINED 1 /* don't pull in MinGW struct stat from wchar.h */ -# endif -#endif - -#include -#include - -#include "bacula.h" +#include "common.h" +#include "tray-ui.h" #include "tray_conf.h" -#include "jcr.h" - - -struct job_defaults { - QString job_name; - QString pool_name; - QString messages_name; - QString client_name; - QString store_name; - QString where; - QString level; - QString type; - QString fileset_name; - QString catalog_name; - bool enabled; -}; - -struct resources { - QStringList job_list; - QStringList pool_list; - QStringList client_list; - QStringList storage_list; - QStringList levels; - QStringList fileset_list; - QStringList messages_list; -}; - -enum stateenum { - idle = 0, - running = 1, - warn = 2, - error = 3 -}; -class monitoritem; -int doconnect(monitoritem* item); -void get_list(monitoritem* item, const char *cmd, QStringList &lst); +bool parse_tmon_config(CONFIG *config, const char *configfile, int exit_code); +bool reload(); +void display_error(const char *msg, ...); -class monitoritem { -public: - rescode type; /* R_DIRECTOR, R_CLIENT or R_STORAGE */ - void* resource; /* DIRRES*, CLIENT* or STORE* */ - BSOCK *D_sock; - stateenum state; - stateenum oldstate; - - char *get_name() { - return ((URES*)resource)->hdr.name; - } - - void writecmd(const char* command) { - if (this->D_sock) { - this->D_sock->msglen = pm_strcpy(&this->D_sock->msg, command); - this->D_sock->send(); - } - } - - bool get_job_defaults(struct job_defaults &job_defs) - { - int stat; - char *def; - BSOCK *dircomm; - bool rtn = false; - QString scmd = QString(".defaults job=\"%1\"").arg(job_defs.job_name); - - if (job_defs.job_name == "") { - return rtn; - } - - if (!doconnect(this)) { - return rtn; - } - dircomm = this->D_sock; - dircomm->fsend("%s", scmd.toUtf8().data()); - - while ((stat = dircomm->recv()) > 0) { - def = strchr(dircomm->msg, '='); - if (!def) { - continue; - } - /* Pointer to default value */ - *def++ = 0; - strip_trailing_junk(def); - - if (strcmp(dircomm->msg, "job") == 0) { - if (strcmp(def, job_defs.job_name.toUtf8().data()) != 0) { - goto bail_out; - } - continue; - } - if (strcmp(dircomm->msg, "pool") == 0) { - job_defs.pool_name = def; - continue; - } - if (strcmp(dircomm->msg, "messages") == 0) { - job_defs.messages_name = def; - continue; - } - if (strcmp(dircomm->msg, "client") == 0) { - job_defs.client_name = def; - continue; - } - if (strcmp(dircomm->msg, "storage") == 0) { - job_defs.store_name = def; - continue; - } - if (strcmp(dircomm->msg, "where") == 0) { - job_defs.where = def; - continue; - } - if (strcmp(dircomm->msg, "level") == 0) { - job_defs.level = def; - continue; - } - if (strcmp(dircomm->msg, "type") == 0) { - job_defs.type = def; - continue; - } - if (strcmp(dircomm->msg, "fileset") == 0) { - job_defs.fileset_name = def; - continue; - } - if (strcmp(dircomm->msg, "catalog") == 0) { - job_defs.catalog_name = def; - continue; - } - if (strcmp(dircomm->msg, "enabled") == 0) { - job_defs.enabled = *def == '1' ? true : false; - continue; - } - } - rtn = true; - /* Fall through wanted */ - bail_out: - return rtn; - } -}; - -#endif /* TRAY_MONITOR_H */ +#endif diff --git a/bacula/src/qt-console/tray-monitor/tray-monitor.pro.in b/bacula/src/qt-console/tray-monitor/tray-monitor.pro.in index 924267c175..92247c0ca7 100644 --- a/bacula/src/qt-console/tray-monitor/tray-monitor.pro.in +++ b/bacula/src/qt-console/tray-monitor/tray-monitor.pro.in @@ -26,6 +26,7 @@ confs.commands = ./install_conf_file TEMPLATE = app TARGET = bacula-tray-monitor +QMAKE_EXTRA_TARGETS += depend DEPENDPATH += . INCLUDEPATH += ../.. . LIBTOOL_LINK = @QMAKE_LIBTOOL@ --silent --tag=CXX --mode=link @@ -33,6 +34,10 @@ LIBTOOL_INSTALL = @QMAKE_LIBTOOL@ --silent --mode=install QMAKE_LINK = $${LIBTOOL_LINK} $(CXX) QMAKE_INSTALL_PROGRAM = $${LIBTOOL_INSTALL} install -m @SBINPERM@ -p QMAKE_CLEAN += obj/* .libs/* bacula-tray-monitor release/bacula-tray-monitor +QMAKE_CXXFLAGS += -DTRAY_MONITOR +QMAKE_CFLAGS += -DTRAY_MONITOR + +INSTALLS = bins confs RESOURCES = ../main.qrc MOC_DIR = moc @@ -40,7 +45,9 @@ OBJECTS_DIR = obj UI_DIR = ui # Main directory -HEADERS += tray_conf.h tray-monitor.h tray-ui.h -SOURCES += authenticate.cpp tray_conf.cpp tray-monitor.cpp +HEADERS += tray-monitor.h tray_conf.h tray-ui.h fdstatus.h task.h ../util/fmtwidgetitem.h dirstatus.h conf.h sdstatus.h runjob.h status.h +SOURCES += tray-monitor.cpp tray_conf.cpp fdstatus.cpp task.cpp authenticate.cpp ../util/fmtwidgetitem.cpp dirstatus.cpp sdstatus.cpp conf.cpp runjob.cpp status.cpp common.h + +FORMS += fd-monitor.ui dir-monitor.ui sd-monitor.ui main-conf.ui res-conf.ui run.ui -FORMS += ../run/run.ui +TRANSLATIONS += ts/tm_fr.ts ts/tm_de.ts ts/tm_ja.ts diff --git a/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw32.in b/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw32.in index cb212b2c22..763c03244a 100644 --- a/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw32.in +++ b/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw32.in @@ -26,6 +26,7 @@ confs.commands = ./install_conf_file TEMPLATE = app TARGET = bacula-tray-monitor +QMAKE_EXTRA_TARGETS += depend DEPENDPATH += . INCLUDEPATH += ../.. . LIBTOOL_LINK = @QMAKE_LIBTOOL@ --silent --tag=CXX --mode=link @@ -33,6 +34,9 @@ LIBTOOL_INSTALL = @QMAKE_LIBTOOL@ --silent --mode=install QMAKE_LINK = $${LIBTOOL_LINK} $(CXX) QMAKE_INSTALL_PROGRAM = $${LIBTOOL_INSTALL} install -m @SBINPERM@ -p QMAKE_CLEAN += .libs/* bacula-tray-monitor release/bacula-tray-monitor +QMAKE_CXXFLAGS += -DTRAY_MONITOR +QMAKE_CFLAGS += -DTRAY_MONITOR + RESOURCES = ../main.qrc MOC_DIR = moc32 @@ -49,7 +53,9 @@ QMAKE_LIB = i686-w64-mingw32-ar -ru QMAKE_RC = i686-w64-mingw32-windres # Main directory -HEADERS += tray_conf.h tray-monitor.h tray-ui.h -SOURCES += authenticate.cpp tray_conf.cpp tray-monitor.cpp +HEADERS += tray-monitor.h tray_conf.h tray-ui.h fdstatus.h task.h ../util/fmtwidgetitem.h dirstatus.h conf.h sdstatus.h runjob.h status.h +SOURCES += tray-monitor.cpp tray_conf.cpp fdstatus.cpp task.cpp authenticate.cpp ../util/fmtwidgetitem.cpp dirstatus.cpp sdstatus.cpp conf.cpp runjob.cpp status.cpp common.h + +FORMS += fd-monitor.ui dir-monitor.ui sd-monitor.ui main-conf.ui res-conf.ui run.ui -FORMS += ../run/run.ui +TRANSLATIONS += ts/tm_fr.ts ts/tm_de.ts ts/tm_ja.ts diff --git a/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw64.in b/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw64.in index a52c640c94..39957e67f9 100644 --- a/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw64.in +++ b/bacula/src/qt-console/tray-monitor/tray-monitor.pro.mingw64.in @@ -26,6 +26,7 @@ confs.commands = ./install_conf_file TEMPLATE = app TARGET = bacula-tray-monitor +QMAKE_EXTRA_TARGETS += depend DEPENDPATH += . INCLUDEPATH += ../.. . LIBTOOL_LINK = @QMAKE_LIBTOOL@ --silent --tag=CXX --mode=link @@ -33,6 +34,9 @@ LIBTOOL_INSTALL = @QMAKE_LIBTOOL@ --silent --mode=install QMAKE_LINK = $${LIBTOOL_LINK} $(CXX) QMAKE_INSTALL_PROGRAM = $${LIBTOOL_INSTALL} install -m @SBINPERM@ -p QMAKE_CLEAN += .libs/* bacula-tray-monitor release/bacula-tray-monitor +QMAKE_CXXFLAGS += -DTRAY_MONITOR +QMAKE_CFLAGS += -DTRAY_MONITOR + RESOURCES = ../main.qrc MOC_DIR = moc64 @@ -49,7 +53,9 @@ QMAKE_LIB = x86_64-w64-mingw32-ar -ru QMAKE_RC = x86_64-w64-mingw32-windres # Main directory -HEADERS += tray_conf.h tray-monitor.h tray-ui.h -SOURCES += authenticate.cpp tray_conf.cpp tray-monitor.cpp +HEADERS += tray-monitor.h tray_conf.h tray-ui.h fdstatus.h task.h ../util/fmtwidgetitem.h dirstatus.h conf.h sdstatus.h runjob.h status.h +SOURCES += tray-monitor.cpp tray_conf.cpp fdstatus.cpp task.cpp authenticate.cpp ../util/fmtwidgetitem.cpp dirstatus.cpp sdstatus.cpp conf.cpp runjob.cpp status.cpp common.h + +FORMS += fd-monitor.ui dir-monitor.ui sd-monitor.ui main-conf.ui res-conf.ui run.ui -FORMS += ../run/run.ui +TRANSLATIONS += ts/tm_fr.ts ts/tm_de.ts ts/tm_ja.ts diff --git a/bacula/src/qt-console/tray-monitor/tray-ui.h b/bacula/src/qt-console/tray-monitor/tray-ui.h index 515bf0f460..af65c6d8e5 100644 --- a/bacula/src/qt-console/tray-monitor/tray-ui.h +++ b/bacula/src/qt-console/tray-monitor/tray-ui.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,13 +20,8 @@ #ifndef TRAYUI_H #define TRAYUI_H -#ifdef HAVE_WIN32 -# ifndef _STAT_DEFINED -# define _STAT_DEFINED 1 /* don't pull in MinGW struct stat from wchar.h */ -# endif -#endif -//#include "winhdrs.h" +#include "common.h" #include #include #include @@ -47,134 +42,17 @@ #include #include #include +#include -#include "version.h" -#include "ui_run.h" -#include "tray-monitor.h" +#include "fdstatus.h" +#include "sdstatus.h" +#include "dirstatus.h" +#include "conf.h" +#include "runjob.h" -class RunDlg: public QDialog, public Ui::runForm -{ - Q_OBJECT - -public: - monitoritem *item; - - void fill(QComboBox *cb, QStringList &lst) { - if (lst.length()) { - cb->addItems(lst); - } else { - cb->setEnabled(false); - } - } - RunDlg(monitoritem *i) { - struct resources res; - struct job_defaults jdefault; - QDateTime dt; - item = i; - - qDebug() << "start getting elements"; - get_list(item, ".jobs type=B", res.job_list); - - if (res.job_list.length() == 0) { - QMessageBox msgBox; - msgBox.setText("This restricted console doesn't have access to Backup jobs"); - msgBox.setIcon(QMessageBox::Warning); - msgBox.exec(); - this->deleteLater(); - return; - } - - get_list(item, ".pools", res.pool_list); - get_list(item, ".clients", res.client_list); - get_list(item, ".storage", res.storage_list); - res.levels << "Full" << "Incremental" << "Differential"; - get_list(item, ".filesets", res.fileset_list); - get_list(item, ".messages", res.messages_list); - - setupUi(this); - - qDebug() << " -> done"; - label_5->setVisible(false); - bootstrap->setVisible(false); - jobCombo->addItems(res.job_list); - fill(filesetCombo, res.fileset_list); - fill(levelCombo, res.levels); - fill(clientCombo, res.client_list); - fill(poolCombo, res.pool_list); - fill(storageCombo, res.storage_list); - dateTimeEdit->setDisplayFormat("yyyy-MM-dd hh:mm:ss"); - dateTimeEdit->setDateTime(dt.currentDateTime()); - fill(messagesCombo, res.messages_list); - messagesCombo->setEnabled(false); - job_name_change(0); - connect(jobCombo, SIGNAL(currentIndexChanged(int)), this, SLOT(job_name_change(int))); - connect(cancelButton, SIGNAL(pressed()), this, SLOT(deleteLater())); - connect(okButton, SIGNAL(pressed()), this, SLOT(okButtonPushed())); - show(); - } +void display_error(const char *fmt, ...); -private slots: - - void okButtonPushed() - { - QString cmd; - cmd = "run"; - if (jobCombo->isEnabled()) { - cmd += " job=\"" + jobCombo->currentText() + "\"" ; - } - if (filesetCombo->isEnabled()) { - cmd += " fileset=\"" + filesetCombo->currentText() + "\""; - } - cmd += " level=\"" + levelCombo->currentText() + "\""; - if (clientCombo->isEnabled()) { - cmd += " client=\"" + clientCombo->currentText() + "\"" ; - } - if (poolCombo->isEnabled()) { - cmd += " pool=\"" + poolCombo->currentText() + "\""; - } - if (storageCombo->isEnabled()) { - cmd += " storage=\"" + storageCombo->currentText() + "\""; - } - cmd += " priority=\"" + QString().setNum(prioritySpin->value()) + "\""; - cmd += " when=\"" + dateTimeEdit->dateTime().toString("yyyy-MM-dd hh:mm:ss") + "\""; -#ifdef xxx - " messages=\"" << messagesCombo->currentText() << "\""; - /* FIXME when there is an option to modify the messages resoruce associated - * with a job */ -#endif - cmd += " yes"; - qDebug() << cmd; - item->D_sock->fsend("%s", cmd.toUtf8().data()); - QString output; - while(item->D_sock->recv() >= 0) {output += item->D_sock->msg;} - QMessageBox msgBox; - msgBox.setText(output); - msgBox.exec(); - deleteLater(); - } - - void job_name_change(int) - { - job_defaults job_defs; - job_defs.job_name = jobCombo->currentText(); - - if (item->get_job_defaults(job_defs)) { - typeLabel->setText("

"+job_defs.type+"

"); - filesetCombo->setCurrentIndex(filesetCombo->findText(job_defs.fileset_name, Qt::MatchExactly)); - levelCombo->setCurrentIndex(levelCombo->findText(job_defs.level, Qt::MatchExactly)); - clientCombo->setCurrentIndex(clientCombo->findText(job_defs.client_name, Qt::MatchExactly)); - poolCombo->setCurrentIndex(poolCombo->findText(job_defs.pool_name, Qt::MatchExactly)); - storageCombo->setCurrentIndex(storageCombo->findText(job_defs.store_name, Qt::MatchExactly)); - messagesCombo->setCurrentIndex(messagesCombo->findText(job_defs.messages_name, Qt::MatchExactly)); - - } else { - - } - } -}; - -void refresh_item(); -void dotest(); +int tls_pem_callback(char *buf, int size, const void *userdata); class TrayUI: public QMainWindow { @@ -184,55 +62,74 @@ public: QWidget *centralwidget; QTabWidget *tabWidget; QStatusBar *statusbar; - QHash hash; - monitoritem* director; QSystemTrayIcon *tray; QSpinBox *spinRefresh; QTimer *timer; - - QPlainTextEdit *getTextEdit(char *title) - { - return hash.value(QString(title)); - } - - void clearText(char *title) + bool have_systray; + + TrayUI(): + QMainWindow(), + tabWidget(NULL), + statusbar(NULL), + tray(NULL), + spinRefresh(NULL), + timer(NULL), + have_systray(QSystemTrayIcon::isSystemTrayAvailable()) + { + }; + + ~TrayUI() { + }; + void addTab(RESMON *r) { - QPlainTextEdit *w = getTextEdit(title); - if (!w) { - return; + QWidget *tab; + QString t = QString(r->hdr.name); + if (r->tls_enable) { + char buf[512]; + /* Generate passphrase prompt */ + bsnprintf(buf, sizeof(buf), "Passphrase for \"%s\" TLS private key: ", r->hdr.name); + + /* Initialize TLS context: + * Args: CA certfile, CA certdir, Certfile, Keyfile, + * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer + */ + r->tls_ctx = new_tls_context(r->tls_ca_certfile, + r->tls_ca_certdir, r->tls_certfile, + r->tls_keyfile, tls_pem_callback, &buf, NULL, true); + + if (!r->tls_ctx) { + display_error(_("Failed to initialize TLS context for \"%s\".\n"), r->hdr.name); + } } - w->clear(); - } - - void appendText(char *title, char *line) - { - QPlainTextEdit *w = getTextEdit(title); - if (!w) { + switch(r->type) { + case R_CLIENT: + tab = new FDStatus(r); + break; + case R_STORAGE: + tab = new SDStatus(r); + break; + case R_DIRECTOR: + tab = new DIRStatus(r); + break; + default: return; } - w->appendPlainText(QString(line)); + tabWidget->setUpdatesEnabled(false); + tabWidget->addTab(tab, t); + tabWidget->setUpdatesEnabled(true); } - - void addDirector(monitoritem *item) + void clearTabs() { - director = item; - } - - void addTab(char *title) - { - QString t = QString(title); - QWidget *tab = new QWidget(); - QVBoxLayout *vLayout = new QVBoxLayout(tab); - QPlainTextEdit *plainTextEdit = new QPlainTextEdit(tab); - plainTextEdit->setObjectName(t); - plainTextEdit->setReadOnly(true); - plainTextEdit->setFont(QFont("courier")); - vLayout->addWidget(plainTextEdit); - hash.insert(t, plainTextEdit); - tabWidget->addTab(tab, t); + tabWidget->setUpdatesEnabled(false); + for(int i = tabWidget->count() - 1; i >= 0; i--) { + QWidget *w = tabWidget->widget(i); + tabWidget->removeTab(i); + delete w; + } + tabWidget->setUpdatesEnabled(true); + tabWidget->update(); } - void startTimer() { if (!timer) { @@ -241,11 +138,10 @@ public: } timer->start(spinRefresh->value()*1000); } - - void setupUi(QMainWindow *TrayMonitor) + void setupUi(QMainWindow *TrayMonitor, MONITOR *mon) { + QPushButton *menubp = NULL; timer = NULL; - director = NULL; if (TrayMonitor->objectName().isEmpty()) TrayMonitor->setObjectName(QString::fromUtf8("TrayMonitor")); TrayMonitor->setWindowIcon(QIcon(":/images/cartridge1.png")); @@ -263,9 +159,14 @@ public: QDialogButtonBox *buttonBox = new QDialogButtonBox(centralwidget); buttonBox->setObjectName(QString::fromUtf8("buttonBox")); - buttonBox->setStandardButtons(QDialogButtonBox::Close); - connect(buttonBox, SIGNAL(rejected()), this, SLOT(cb_show())); - + if (have_systray) { + buttonBox->setStandardButtons(QDialogButtonBox::Close); + connect(buttonBox, SIGNAL(rejected()), this, SLOT(cb_show())); + } else { + /* Here we can display something else, now it's just a simple menu */ + menubp = new QPushButton(tr("&Options")); + buttonBox->addButton(menubp, QDialogButtonBox::ActionRole); + } TrayMonitor->setCentralWidget(centralwidget); statusbar = new QStatusBar(TrayMonitor); statusbar->setObjectName(QString::fromUtf8("statusbar")); @@ -284,12 +185,12 @@ public: spinRefresh->setMinimum(1); spinRefresh->setMaximum(600); spinRefresh->setSingleStep(10); - spinRefresh->setValue(60); + spinRefresh->setValue(mon?mon->RefreshInterval:60); hLayout->addWidget(spinRefresh); hLayout->addWidget(buttonBox); verticalLayout->addLayout(hLayout); - + //QSystemTrayIcon::isSystemTrayAvailable tray = new QSystemTrayIcon(TrayMonitor); QMenu* stmenu = new QMenu(TrayMonitor); @@ -305,28 +206,49 @@ public: QAction* actRun = new QAction(QApplication::translate("TrayMonitor", "Run...", 0, QApplication::UnicodeUTF8),TrayMonitor); +/* Not yet ready + * QAction* actRes = new QAction(QApplication::translate("TrayMonitor", + * "Restore...", + * 0, QApplication::UnicodeUTF8),TrayMonitor); + */ + QAction* actConf = new QAction(QApplication::translate("TrayMonitor", + "Configure...", + 0, QApplication::UnicodeUTF8),TrayMonitor); stmenu->addAction(actShow); stmenu->addAction(actRun); + //stmenu->addAction(actRes); + stmenu->addSeparator(); + stmenu->addAction(actConf); + stmenu->addSeparator(); stmenu->addAction(actAbout); stmenu->addSeparator(); stmenu->addAction(actQuit); connect(actRun, SIGNAL(triggered()), this, SLOT(cb_run())); connect(actShow, SIGNAL(triggered()), this, SLOT(cb_show())); + connect(actConf, SIGNAL(triggered()), this, SLOT(cb_conf())); + //connect(actRes, SIGNAL(triggered()), this, SLOT(cb_restore())); connect(actQuit, SIGNAL(triggered()), this, SLOT(cb_quit())); connect(actAbout, SIGNAL(triggered()), this, SLOT(cb_about())); connect(spinRefresh, SIGNAL(valueChanged(int)), this, SLOT(cb_refresh(int))); connect(tray, SIGNAL(activated(QSystemTrayIcon::ActivationReason)), this, SLOT(cb_trayIconActivated(QSystemTrayIcon::ActivationReason))); - tray->setContextMenu(stmenu); + QIcon icon(":/images/cartridge1.png"); tray->setIcon(icon); tray->setToolTip(QString("Bacula Tray Monitor")); tray->show(); - retranslateUi(TrayMonitor); QMetaObject::connectSlotsByName(TrayMonitor); + startTimer(); + + /* When we don't have the systemtray, we keep the menu, but disabled */ + if (!have_systray) { + actShow->setEnabled(false); + menubp->setMenu(stmenu); + TrayMonitor->show(); + } } // setupUi void retranslateUi(QMainWindow *TrayMonitor) @@ -347,18 +269,105 @@ private slots: void cb_about() { QMessageBox::about(this, "Bacula Tray Monitor", "Bacula Tray Monitor\n" - "For more information, see: www.baculasystems.com\n" - "Copyright (C) 1999-2015, Kern Sibbald.\n" - "License AGPLv3 see LICENSE."); + "For more information, see: www.bacula.org\n" + "Copyright (C) 1999-2017, Kern Sibbald.\n" + "All rights reserved."); + } + RESMON *get_director() { + QStringList dirs; + RESMON *d, *director=NULL; + bool ok; + foreach_res(d, R_DIRECTOR) { + if (!director) { + director = d; + } + dirs << QString(d->hdr.name); + } + foreach_res(d, R_CLIENT) { + if (d->use_remote) { + if (!director) { + director = d; + } + dirs << QString(d->hdr.name); + } + } + if (dirs.count() > 1) { + /* TODO: Set Modal attribute */ + QString dir = QInputDialog::getItem(this, _("Select a Director"), "Director:", dirs, 0, false, &ok, 0); + if (!ok) { + return NULL; + } + if (ok && !dir.isEmpty()) { + char *p = dir.toUtf8().data(); + foreach_res(d, R_DIRECTOR) { + if (strcmp(p, d->hdr.name) == 0) { + director = d; + break; + } + } + foreach_res(d, R_CLIENT) { + if (strcmp(p, d->hdr.name) == 0) { + director = d; + break; + } + } + } + } + if (dirs.count() == 0 || director == NULL) { + /* We need the proxy feature */ + display_error("No Director defined"); + return NULL; + } + return director; } - void cb_run() { - if (director) { - RunDlg *runbox = new RunDlg(director); - runbox->show(); + RESMON *dir = get_director(); + if (!dir) { + return; } + task *t = new task(); + connect(t, SIGNAL(done(task *)), this, SLOT(run_job(task *)), Qt::QueuedConnection); + t->init(dir, TASK_RESOURCES); + dir->wrk->queue(t); } + void refresh_item() { + /* Probably do only the first one */ + int oldnbjobs = 0; + + for (int i=tabWidget->count() - 1; i >= 0; i--) { + ResStatus *s = (ResStatus *) tabWidget->widget(i); + if (s->res->use_monitor) { + s->res->mutex->lock(); + if (s->res->running_jobs) { + oldnbjobs += s->res->running_jobs->size(); + } + s->res->mutex->unlock(); + } + if (isVisible() || s->res->use_monitor) { + s->doUpdate(); + } + } + /* We need to find an other way to compute running jobs */ + if (oldnbjobs) { + QString q; + tray->setIcon(QIcon(":/images/R.png")); + tray->setToolTip(q.sprintf("Bacula Tray Monitor - %d job%s running", oldnbjobs, oldnbjobs>1?"s":"")); + //tray->showMessage(); Can use this function to display a popup + } else { + tray->setIcon(QIcon(":/images/cartridge1.png")); + tray->setToolTip("Bacula Tray Monitor"); + } + } + void cb_conf() { + new Conf(); + } + void cb_restore() { + RESMON *dir = get_director(); + if (!dir) { + return; + } + } void cb_trayIconActivated(QSystemTrayIcon::ActivationReason r) { if (r == QSystemTrayIcon::Trigger) { cb_show(); @@ -366,11 +375,7 @@ private slots: } void refresh_screen() { -// qDebug() << "refresh_screen()"; - if (isVisible()) { - refresh_item(); -// qDebug() << " -> OK"; - } + refresh_item(); } void cb_show() { @@ -381,6 +386,17 @@ private slots: show(); } } +public slots: + void task_done(task *t) { + Dmsg0(0, "Task done!\n"); + t->deleteLater(); + }; + void run_job(task *t) { + Dmsg0(0, "Task done!\n"); + RESMON *dir = t->res; + t->deleteLater(); + new RunJob(dir); + }; }; diff --git a/bacula/src/qt-console/tray-monitor/tray_conf.cpp b/bacula/src/qt-console/tray-monitor/tray_conf.cpp index 6fab08f6ea..76809ff895 100644 --- a/bacula/src/qt-console/tray-monitor/tray_conf.cpp +++ b/bacula/src/qt-console/tray-monitor/tray_conf.cpp @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -39,17 +39,19 @@ * */ -#include "bacula.h" +#include "common.h" #include "tray_conf.h" +worker *worker_start(); +void worker_stop(worker *); + /* Define the first and last resource ID record * types. Note, these should be unique for each * daemon though not a requirement. */ int32_t r_first = R_FIRST; int32_t r_last = R_LAST; -static RES *sres_head[R_LAST - R_FIRST + 1]; -RES **res_head = sres_head; +RES_HEAD **res_head; /* We build the current resource here as we are * scanning the resource configuration definition, @@ -70,24 +72,32 @@ int32_t res_all_size = sizeof(res_all); * name handler value code flags default_value */ static RES_ITEM mon_items[] = { - {"name", store_name, ITEM(res_monitor.hdr.name), 0, ITEM_REQUIRED, 0}, - {"description", store_str, ITEM(res_monitor.hdr.desc), 0, 0, 0}, + {"Name", store_name, ITEM(res_monitor.hdr.name), 0, ITEM_REQUIRED, 0}, + {"Description", store_str, ITEM(res_monitor.hdr.desc), 0, 0, 0}, {"requiressl", store_bool, ITEM(res_monitor.require_ssl), 1, ITEM_DEFAULT, 0}, - {"password", store_password, ITEM(res_monitor.password), 0, ITEM_REQUIRED, 0}, - {"refreshinterval", store_time,ITEM(res_monitor.RefreshInterval), 0, ITEM_DEFAULT, 60}, - {"fdconnecttimeout", store_time,ITEM(res_monitor.FDConnectTimeout), 0, ITEM_DEFAULT, 10}, - {"sdconnecttimeout", store_time,ITEM(res_monitor.SDConnectTimeout), 0, ITEM_DEFAULT, 10}, - {"dirconnecttimeout", store_time,ITEM(res_monitor.DIRConnectTimeout), 0, ITEM_DEFAULT, 10}, + {"RefreshInterval", store_time,ITEM(res_monitor.RefreshInterval), 0, ITEM_DEFAULT, 60}, + {"CommCompression", store_bool, ITEM(res_monitor.comm_compression), 0, ITEM_DEFAULT, true}, + {"CommandDirectory", store_dir, ITEM(res_monitor.command_dir), 0, 0, 0}, + {"DisplayAdvancedOptions", store_bool, ITEM(res_monitor.display_advanced_options), 0, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; /* Director's that we can contact */ static RES_ITEM dir_items[] = { - {"name", store_name, ITEM(res_dir.hdr.name), 0, ITEM_REQUIRED, 0}, - {"description", store_str, ITEM(res_dir.hdr.desc), 0, 0, 0}, - {"dirport", store_pint32, ITEM(res_dir.DIRport), 0, ITEM_DEFAULT, 9101}, - {"address", store_str, ITEM(res_dir.address), 0, ITEM_REQUIRED, 0}, - {"enablessl", store_bool, ITEM(res_dir.enable_ssl), 1, ITEM_DEFAULT, 0}, + {"Name", store_name, ITEM(res_main.hdr.name), 0, ITEM_REQUIRED, 0}, + {"Description", store_str, ITEM(res_main.hdr.desc), 0, 0, 0}, + {"Port", store_pint32, ITEM(res_main.port), 0, ITEM_DEFAULT, 9101}, + {"Address", store_str, ITEM(res_main.address), 0, ITEM_REQUIRED, 0}, + {"Password", store_password, ITEM(res_main.password), 0, ITEM_REQUIRED, 0}, + {"Monitor", store_bool, ITEM(res_main.use_monitor), 0, ITEM_DEFAULT, 0}, + {"ConnectTimeout", store_time,ITEM(res_main.connect_timeout), 0, ITEM_DEFAULT, 10}, + {"UseSetIp", store_bool, ITEM(res_main.use_setip), 0, 0, 0}, + {"TlsEnable", store_bool, ITEM(res_main.tls_enable), 0, 0, 0}, + {"TlsCaCertificateFile", store_dir, ITEM(res_main.tls_ca_certfile), 0, 0, 0}, + {"TlsCaCertificateDir", store_dir, ITEM(res_main.tls_ca_certdir), 0, 0, 0}, + {"TlsCertificate", store_dir, ITEM(res_main.tls_certfile), 0, 0, 0}, + {"TlsKey", store_dir, ITEM(res_main.tls_keyfile), 0, 0, 0}, + {NULL, NULL, {0}, 0, 0, 0} }; @@ -98,12 +108,19 @@ static RES_ITEM dir_items[] = { */ static RES_ITEM cli_items[] = { - {"name", store_name, ITEM(res_client.hdr.name), 0, ITEM_REQUIRED, 0}, - {"description", store_str, ITEM(res_client.hdr.desc), 0, 0, 0}, - {"address", store_str, ITEM(res_client.address), 0, ITEM_REQUIRED, 0}, - {"fdport", store_pint32, ITEM(res_client.FDport), 0, ITEM_DEFAULT, 9102}, - {"password", store_password, ITEM(res_client.password), 0, ITEM_REQUIRED, 0}, - {"enablessl", store_bool, ITEM(res_client.enable_ssl), 1, ITEM_DEFAULT, 0}, + {"Name", store_name, ITEM(res_main.hdr.name), 0, ITEM_REQUIRED, 0}, + {"Description", store_str, ITEM(res_main.hdr.desc), 0, 0, 0}, + {"Address", store_str, ITEM(res_main.address), 0, ITEM_REQUIRED, 0}, + {"Port", store_pint32, ITEM(res_main.port), 0, ITEM_DEFAULT, 9102}, + {"Password", store_password, ITEM(res_main.password), 0, ITEM_REQUIRED, 0}, + {"ConnectTimeout", store_time,ITEM(res_main.connect_timeout), 0, ITEM_DEFAULT, 10}, + {"Remote", store_bool, ITEM(res_main.use_remote), 0, ITEM_DEFAULT, 0}, + {"Monitor", store_bool, ITEM(res_main.use_monitor), 0, ITEM_DEFAULT, 0}, + {"TlsEnable", store_bool, ITEM(res_main.tls_enable), 0, 0, 0}, + {"TlsCaCertificateFile", store_dir, ITEM(res_main.tls_ca_certfile), 0, 0, 0}, + {"TlsCaCertificateDir", store_dir, ITEM(res_main.tls_ca_certdir), 0, 0, 0}, + {"TlsCertificate", store_dir, ITEM(res_main.tls_certfile), 0, 0, 0}, + {"TlsKey", store_dir, ITEM(res_main.tls_keyfile), 0, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -112,21 +129,18 @@ static RES_ITEM cli_items[] = { * name handler value code flags default_value */ static RES_ITEM store_items[] = { - {"name", store_name, ITEM(res_store.hdr.name), 0, ITEM_REQUIRED, 0}, - {"description", store_str, ITEM(res_store.hdr.desc), 0, 0, 0}, - {"sdport", store_pint32, ITEM(res_store.SDport), 0, ITEM_DEFAULT, 9103}, - {"address", store_str, ITEM(res_store.address), 0, ITEM_REQUIRED, 0}, - {"sdaddress", store_str, ITEM(res_store.address), 0, 0, 0}, - {"password", store_password, ITEM(res_store.password), 0, ITEM_REQUIRED, 0}, - {"sdpassword", store_password, ITEM(res_store.password), 0, 0, 0}, - {"enablessl", store_bool, ITEM(res_store.enable_ssl), 1, ITEM_DEFAULT, 0}, - {NULL, NULL, {0}, 0, 0, 0} -}; - -static RES_ITEM con_font_items[] = { - {"name", store_name, ITEM(con_font.hdr.name), 0, ITEM_REQUIRED, 0}, - {"description", store_str, ITEM(con_font.hdr.desc), 0, 0, 0}, - {"font", store_str, ITEM(con_font.fontface), 0, 0, 0}, + {"Name", store_name, ITEM(res_main.hdr.name), 0, ITEM_REQUIRED, 0}, + {"Description", store_str, ITEM(res_main.hdr.desc), 0, 0, 0}, + {"Port", store_pint32, ITEM(res_main.port), 0, ITEM_DEFAULT, 9103}, + {"Address", store_str, ITEM(res_main.address), 0, ITEM_REQUIRED, 0}, + {"Password", store_password, ITEM(res_main.password), 0, ITEM_REQUIRED, 0}, + {"ConnectTimeout", store_time,ITEM(res_main.connect_timeout), 0, ITEM_DEFAULT, 10}, + {"Monitor", store_bool, ITEM(res_main.use_monitor), 0, ITEM_DEFAULT, 0}, + {"TlsEnable", store_bool, ITEM(res_main.tls_enable), 0, 0, 0}, + {"TlsCaCertificateFile", store_dir, ITEM(res_main.tls_ca_certfile), 0, 0, 0}, + {"TlsCaCertificateDir", store_dir, ITEM(res_main.tls_ca_certdir), 0, 0, 0}, + {"TlsCertificate", store_dir, ITEM(res_main.tls_certfile), 0, 0, 0}, + {"TlsKey", store_dir, ITEM(res_main.tls_keyfile), 0, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -144,16 +158,15 @@ RES_TABLE resources[] = { {"director", dir_items, R_DIRECTOR}, {"client", cli_items, R_CLIENT}, {"storage", store_items, R_STORAGE}, - {"consolefont", con_font_items, R_CONSOLE_FONT}, {NULL, NULL, 0} }; /* Dump contents of resource */ -void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), void *sock) +void dump_resource(int type, RES *ares, void sendit(void *sock, const char *fmt, ...), void *sock) { - URES *res = (URES *)reshdr; + RES *next; + URES *res = (URES *)ares; bool recurse = true; - char ed1[100], ed2[100]; if (res == NULL) { sendit(sock, _("No %s resource defined\n"), res_to_str(type)); @@ -165,37 +178,32 @@ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fm } switch (type) { case R_MONITOR: - sendit(sock, _("Monitor: name=%s FDtimeout=%s SDtimeout=%s\n"), - reshdr->name, - edit_uint64(res->res_monitor.FDConnectTimeout, ed1), - edit_uint64(res->res_monitor.SDConnectTimeout, ed2)); + sendit(sock, _("Monitor: name=%s\n"), ares->name); break; case R_DIRECTOR: - sendit(sock, _("Director: name=%s address=%s FDport=%d\n"), - res->res_dir.hdr.name, res->res_dir.address, res->res_dir.DIRport); + sendit(sock, _("Director: name=%s address=%s port=%d\n"), + res->res_main.hdr.name, res->res_main.address, res->res_main.port); break; case R_CLIENT: - sendit(sock, _("Client: name=%s address=%s FDport=%d\n"), - res->res_client.hdr.name, res->res_client.address, res->res_client.FDport); + sendit(sock, _("Client: name=%s address=%s port=%d\n"), + res->res_main.hdr.name, res->res_main.address, res->res_main.port); break; case R_STORAGE: - sendit(sock, _("Storage: name=%s address=%s SDport=%d\n"), - res->res_store.hdr.name, res->res_store.address, res->res_store.SDport); - break; - case R_CONSOLE_FONT: - sendit(sock, _("ConsoleFont: name=%s font face=%s\n"), - reshdr->name, NPRT(res->con_font.fontface)); + sendit(sock, _("Storage: name=%s address=%s port=%d\n"), + res->res_main.hdr.name, res->res_main.address, res->res_main.port); break; default: sendit(sock, _("Unknown resource type %d in dump_resource.\n"), type); break; } - if (recurse && res->res_monitor.hdr.next) { - dump_resource(type, res->res_monitor.hdr.next, sendit, sock); + if (recurse) { + next = GetNextRes(0, (RES *)res); + if (next) { + dump_resource(type, next, sendit, sock); + } } } - /* * Free memory of resource -- called when daemon terminates. * NB, we don't need to worry about freeing any references @@ -205,14 +213,11 @@ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fm */ void free_resource(RES *sres, int type) { - RES *nres; /* next resource if linked */ URES *res = (URES *)sres; if (res == NULL) return; - /* common stuff -- free the resource name and description */ - nres = (RES *)res->res_monitor.hdr.next; if (res->res_monitor.hdr.name) { free(res->res_monitor.hdr.name); } @@ -222,26 +227,57 @@ void free_resource(RES *sres, int type) switch (type) { case R_MONITOR: - break; - case R_CLIENT: - if (res->res_client.address) { - free(res->res_client.address); + if (res->res_monitor.password) { + free(res->res_monitor.password); } - if (res->res_client.password) { - free(res->res_client.password); + if (res->res_monitor.command_dir) { + free(res->res_monitor.command_dir); } break; + case R_DIRECTOR: + case R_CLIENT: case R_STORAGE: - if (res->res_store.address) { - free(res->res_store.address); + delete res->res_main.mutex; + free_bsock(res->res_main.bs); + if (res->res_main.wrk) { + worker_stop(res->res_main.wrk); + res->res_main.wrk = NULL; } - if (res->res_store.password) { - free(res->res_store.password); + if (res->res_main.address) { + free(res->res_main.address); } - break; - case R_CONSOLE_FONT: - if (res->con_font.fontface) { - free(res->con_font.fontface); + if (res->res_main.tls_ctx) { + free_tls_context(res->res_main.tls_ctx); + } + if (res->res_main.tls_ca_certfile) { + free(res->res_main.tls_ca_certfile); + } + if (res->res_main.tls_ca_certdir) { + free(res->res_main.tls_ca_certdir); + } + if (res->res_main.tls_certfile) { + free(res->res_main.tls_certfile); + } + if (res->res_main.tls_keyfile) { + free(res->res_main.tls_keyfile); + } + if (res->res_main.jobs) { + delete res->res_main.jobs; + } + if (res->res_main.clients) { + delete res->res_main.clients; + } + if (res->res_main.filesets) { + delete res->res_main.filesets; + } + if (res->res_main.pools) { + delete res->res_main.pools; + } + if (res->res_main.storages) { + delete res->res_main.storages; + } + if (res->res_main.running_jobs) { + delete res->res_main.terminated_jobs; } break; default: @@ -252,9 +288,6 @@ void free_resource(RES *sres, int type) if (res) { free(res); } - if (nres) { - free_resource(nres, type); - } } /* @@ -263,9 +296,8 @@ void free_resource(RES *sres, int type) * pointers because they may not have been defined until * later in pass 1. */ -void save_resource(int type, RES_ITEM *items, int pass) +bool save_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { - URES *res; int rindex = type - r_first; int i, size; int error = 0; @@ -276,13 +308,15 @@ void save_resource(int type, RES_ITEM *items, int pass) for (i=0; items[i].name; i++) { if (items[i].flags & ITEM_REQUIRED) { if (!bit_is_set(i, res_all.res_monitor.hdr.item_present)) { - Emsg2(M_ERROR_TERM, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), - items[i].name, resources[rindex].name); + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + items[i].name, resources[rindex].name); + return false; } } /* If this triggers, take a look at lib/parse_conf.h */ if (i >= MAX_RES_ITEMS) { - Emsg1(M_ERROR_TERM, 0, _("Too many directives in \"%s\" resource\n"), resources[rindex].name); + Mmsg(config->m_errmsg, _("Too many directives in \"%s\" resource\n"), resources[rindex].name); + return false; } } @@ -295,11 +329,10 @@ void save_resource(int type, RES_ITEM *items, int pass) if (pass == 2) { switch (type) { /* Resources not containing a resource */ - case R_MONITOR: - case R_CLIENT: case R_STORAGE: case R_DIRECTOR: - case R_CONSOLE_FONT: + case R_CLIENT: + case R_MONITOR: break; default: Emsg1(M_ERROR, 0, _("Unknown resource type %d in save_resource.\n"), type); @@ -317,7 +350,7 @@ void save_resource(int type, RES_ITEM *items, int pass) free(res_all.res_monitor.hdr.desc); res_all.res_monitor.hdr.desc = NULL; } - return; + return true; } /* @@ -327,17 +360,13 @@ void save_resource(int type, RES_ITEM *items, int pass) case R_MONITOR: size = sizeof(MONITOR); break; - case R_DIRECTOR: - size = sizeof(DIRRES); - break; case R_CLIENT: - size = sizeof(CLIENT); - break; case R_STORAGE: - size = sizeof(STORE); - break; - case R_CONSOLE_FONT: - size = sizeof(CONFONTRES); + case R_DIRECTOR: + // We need to initialize the mutex + res_all.res_main.mutex = new QMutex(); + res_all.res_main.wrk = worker_start(); + size = sizeof(RESMON); break; default: printf(_("Unknown resource type %d in save_resource.\n"), type); @@ -347,33 +376,18 @@ void save_resource(int type, RES_ITEM *items, int pass) } /* Common */ if (!error) { - res = (URES *)malloc(size); - memcpy(res, &res_all, size); - if (!res_head[rindex]) { - res_head[rindex] = (RES *)res; /* store first entry */ - Dmsg3(900, "Inserting first %s res: %s index=%d\n", res_to_str(type), - res->res_monitor.hdr.name, rindex); - } else { - RES *next, *last; - /* Add new res to end of chain */ - for (last=next=res_head[rindex]; next; next=next->next) { - last = next; - if (strcmp(next->name, res->res_monitor.hdr.name) == 0) { - Emsg2(M_ERROR_TERM, 0, - _("Attempt to define second \"%s\" resource named \"%s\" is not permitted.\n"), - resources[rindex].name, res->res_monitor.hdr.name); - } - } - last->next = (RES *)res; - Dmsg4(900, "Inserting %s res: %s index=%d pass=%d\n", res_to_str(type), - res->res_monitor.hdr.name, rindex, pass); + res_all.res_main.type = type; + if (!config->insert_res(rindex, size)) { + return false; } } + return true; } bool parse_tmon_config(CONFIG *config, const char *configfile, int exit_code) { - config->init(configfile, NULL, exit_code, (void *)&res_all, res_all_size, - r_first, r_last, resources, res_head); + config->init(configfile, error_handler, exit_code, + (void *)&res_all, res_all_size, + r_first, r_last, resources, &res_head); return config->parse_config(); } diff --git a/bacula/src/qt-console/tray-monitor/tray_conf.h b/bacula/src/qt-console/tray-monitor/tray_conf.h index 1bfd554337..f2bc4d8d0f 100644 --- a/bacula/src/qt-console/tray-monitor/tray_conf.h +++ b/bacula/src/qt-console/tray-monitor/tray_conf.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -25,6 +25,11 @@ * */ +#ifndef TRAY_CONF +#define TRAY_CONF + +#include "common.h" + /* NOTE: #includes at the end of this file */ /* @@ -52,12 +57,101 @@ enum { R_BACKUP }; -/* Director */ -struct DIRRES { - RES hdr; - uint32_t DIRport; /* UA server port */ - char *address; /* UA server address */ - bool enable_ssl; /* Use SSL */ +struct s_running_job +{ + int32_t Errors; /* FD/SD errors */ + int32_t JobType; + int32_t JobStatus; + int32_t JobLevel; + uint32_t JobId; + uint32_t VolSessionId; + uint32_t VolSessionTime; + uint32_t JobFiles; + uint64_t JobBytes; + uint64_t ReadBytes; + utime_t start_time; + int64_t bytespersec; + int SDtls; + char Client[MAX_NAME_LENGTH]; + char FileSet[MAX_NAME_LENGTH]; + char Storage[MAX_NAME_LENGTH]; + char RStorage[MAX_NAME_LENGTH]; + char sched_time; + char Job[MAX_NAME_LENGTH]; + char CurrentFile[4096]; +}; + +/* forward definition */ +class worker; + +/* Director/Client/Storage */ +struct RESMON { + RES hdr; /* Keep First */ + uint32_t type; /* Keep 2nd R_CLIENT, R_DIRECTOR, R_STORAGE */ + + uint32_t port; /* UA server port */ + char *address; /* UA server address */ + utime_t connect_timeout; /* timeout for connect in seconds */ + char *password; + + bool use_remote; /* Use Client Initiated backup feature */ + bool use_monitor; /* update the status icon with this resource */ + bool use_setip; /* Send setip command before a job */ + + bool tls_enable; /* Enable TLS on all connections */ + char *tls_ca_certfile; /* TLS CA Certificate File */ + char *tls_ca_certdir; /* TLS CA Certificate Directory */ + char *tls_certfile; /* TLS Client Certificate File */ + char *tls_keyfile; /* TLS Client Key File */ + + /* ------------------------------------------------------------ */ + TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ + worker *wrk; /* worker that will handle async op */ + + QMutex *mutex; + BSOCK *bs; + char name[MAX_NAME_LENGTH]; + char version[MAX_NAME_LENGTH]; + char plugins[MAX_NAME_LENGTH]; + char started[32]; /* ISO date */ + char reloaded[32]; /* ISO date */ + int bwlimit; + alist *running_jobs; + dlist *terminated_jobs; + btime_t last_update; + bool new_resource; + bool proxy_sent; + + /* List of resources available */ + alist *jobs; + alist *clients; + alist *filesets; + alist *pools; + alist *storages; + alist *catalogs; + + /* Default value */ + struct { + char *job; + char *client; + char *pool; + char *storage; + char *level; + char *type; + char *fileset; + char *catalog; + int priority; + } defaults; + + /* Information about the job */ + struct { + uint64_t JobBytes; + uint32_t JobFiles; + int CorrJobBytes; + int CorrJobFiles; + int CorrNbJob; + char JobLevel; + } infos; }; /* @@ -65,56 +159,28 @@ struct DIRRES { * */ struct MONITOR { - RES hdr; + RES hdr; /* Keep first */ + int32_t type; /* Keep second */ + + bool comm_compression; /* Enable comm line compression */ bool require_ssl; /* Require SSL for all connections */ + bool display_advanced_options; /* Display advanced options (run options for example) */ MSGS *messages; /* Daemon message handler */ char *password; /* UA server password */ + char *command_dir; /* Where to find Commands */ utime_t RefreshInterval; /* Status refresh interval */ - utime_t FDConnectTimeout; /* timeout for connect in seconds */ - utime_t SDConnectTimeout; /* timeout in seconds */ - utime_t DIRConnectTimeout; /* timeout in seconds */ }; -/* - * Client Resource - * - */ -struct CLIENT { - RES hdr; - - uint32_t FDport; /* Where File daemon listens */ - char *address; - char *password; - bool enable_ssl; /* Use SSL */ -}; - -/* - * Store Resource - * - */ -struct STORE { - RES hdr; - - uint32_t SDport; /* port where Directors connect */ - char *address; - char *password; - bool enable_ssl; /* Use SSL */ -}; - -struct CONFONTRES { - RES hdr; - char *fontface; /* Console Font specification */ -}; - /* Define the Union of all the above * resource structure definitions. */ union URES { MONITOR res_monitor; - DIRRES res_dir; - CLIENT res_client; - STORE res_store; - CONFONTRES con_font; + RESMON res_main; RES hdr; }; + +void error_handler(const char *file, int line, LEX *lc, const char *msg, ...); + +#endif diff --git a/bacula/src/qt-console/util/fmtwidgetitem.cpp b/bacula/src/qt-console/util/fmtwidgetitem.cpp index 1fbc4d3775..4426760b70 100644 --- a/bacula/src/qt-console/util/fmtwidgetitem.cpp +++ b/bacula/src/qt-console/util/fmtwidgetitem.cpp @@ -25,7 +25,7 @@ * */ -#include "bat.h" +#include "../bat.h" #include #include #include diff --git a/bacula/src/stored/Makefile.in b/bacula/src/stored/Makefile.in index 2805480d33..0fdf68f346 100644 --- a/bacula/src/stored/Makefile.in +++ b/bacula/src/stored/Makefile.in @@ -1,5 +1,5 @@ # -# Copyright (C) 2000-2015 by Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @MCOMMON@ @@ -21,73 +21,69 @@ DEBUG=@DEBUG@ GETTEXT_LIBS = @LIBINTL@ +S3_LIBS=@S3_LIBS@ +S3_INC=@S3_INC@ + DB_LIBS=@DB_LIBS@ first_rule: all dummy: +# Bacula SD core objects needed by all executables +SDCORE_OBJS = \ + stored_conf.o global.o + # bacula-sd -SDOBJS = stored.o ansi_label.o vtape_dev.o \ - autochanger.o acquire.o append.o \ - askdir.o authenticate.o \ - block.o block_util.o butil.o dev.o os.o file_dev.o tape_dev.o \ - device.o dircmd.o ebcdic.o fd_cmds.o job.o \ - hello.o \ - label.o lock.o match_bsr.o mount.o parse_bsr.o \ - read.o read_records.o \ - record_read.o record_write.o record_util.o \ - reserve.o scan.o sd_plugins.o \ - spool.o status.o stored_conf.o \ - vbackup.o vol_mgr.o wait.o +SDOBJS = \ + stored.o append.o authenticate.o dircmd.o fd_cmds.o job.o \ + hello.o status.o vbackup.o \ + $(SDCORE_OBJS) + +JSONOBJS = bsdjson.o stored_conf.o # btape -TAPEOBJS = btape.o block.o block_util.o butil.o \ - dev.o os.o file_dev.o tape_dev.o \ - device.o label.o vtape_dev.o \ - lock.o ansi_label.o ebcdic.o \ - autochanger.o acquire.o mount.o record_util.o \ - read_records.o record_read.o record_write.o \ - reserve.o stored_conf.o match_bsr.o parse_bsr.o scan.o \ - sd_plugins.o status.o spool.o vol_mgr.o wait.o +TAPEOBJS = btape.o $(SDCORE_OBJS) # bls -BLSOBJS = bls.o block.o block_util.o butil.o device.o \ - dev.o os.o file_dev.o tape_dev.o label.o match_bsr.o vtape_dev.o \ - ansi_label.o ebcdic.o lock.o \ - autochanger.o acquire.o mount.o parse_bsr.o \ - record_read.o record_write.o record_util.o \ - read_records.o reserve.o scan.o stored_conf.o spool.o \ - sd_plugins.o status.o vol_mgr.o wait.o +BLSOBJS = bls.o $(SDCORE_OBJS) # bextract -BEXTOBJS = bextract.o block.o block_util.o device.o \ - dev.o os.o file_dev.o tape_dev.o label.o vtape_dev.o \ - ansi_label.o ebcdic.o lock.o \ - autochanger.o acquire.o mount.o match_bsr.o parse_bsr.o butil.o \ - read_records.o record_read.o record_write.o record_util.o \ - reserve.o scan.o stored_conf.o spool.o \ - sd_plugins.o status.o vol_mgr.o wait.o +BEXTOBJS = bextract.o $(SDCORE_OBJS) # bscan -SCNOBJS = bscan.o block.o block_util.o device.o \ - dev.o os.o file_dev.o tape_dev.o label.o vtape_dev.o \ - ansi_label.o ebcdic.o lock.o \ - autochanger.o acquire.o mount.o \ - record_read.o record_write.o read_records.o record_util.o \ - match_bsr.o parse_bsr.o \ - butil.o scan.o reserve.o stored_conf.o spool.o \ - sd_plugins.o status.o vol_mgr.o wait.o +SCNOBJS = bscan.o $(SDCORE_OBJS) # bcopy -COPYOBJS = bcopy.o block.o block_util.o device.o \ - dev.o os.o file_dev.o tape_dev.o label.o vtape_dev.o \ - ansi_label.o ebcdic.o lock.o \ - autochanger.o acquire.o mount.o \ - record_read.o record_write.o read_records.o record_util.o \ - match_bsr.o parse_bsr.o butil.o reserve.o \ - sd_plugins.o scan.o status.o stored_conf.o spool.o \ - vol_mgr.o wait.o +COPYOBJS = bcopy.o $(SDCORE_OBJS) + +ALIGNED_SRCS = \ + aligned_dev.c aligned_read.c aligned_write.c + +ALIGNED_OBJS = $(ALIGNED_SRCS:.c=.o) +ALIGNED_LOBJS = $(ALIGNED_SRCS:.c=.lo) + +CLOUD_SRCS = \ + cloud_dev.c cloud_parts.c cloud_transfer_mgr.c s3_driver.c file_driver.c +CLOUD_OBJS = $(CLOUD_SRCS:.c=.o) +CLOUD_LOBJS = $(CLOUD_SRCS:.c=.lo) + +# cloud_test +CLOUDTESTOBJS = cloud_test.o $(SDCORE_OBJS) + +# libbacsd objects +LIBBACSD_SRCS = \ + acquire.c ansi_label.c askdir.c autochanger.c \ + block.c block_util.c butil.c dev.c device.c ebcdic.c \ + init_dev.c label.c lock.c match_bsr.c mount.c \ + null_dev.c os.c parse_bsr.c read.c read_records.c \ + record_read.c record_util.c record_write.c reserve.c \ + scan.c sd_plugins.c spool.c tape_alert.c vol_mgr.c wait.c \ + fifo_dev.c file_dev.c tape_dev.c vtape_dev.c + +LIBBACSD_OBJS = $(LIBBACSD_SRCS:.c=.o) +LIBBACSD_LOBJS = $(LIBBACSD_SRCS:.c=.lo) +LIBBACSD_LT_RELEASE = @LIBBAC_LT_RELEASE@ # these are the objects that are changed by the .configure process @@ -98,8 +94,10 @@ ZLIBS=@ZLIBS@ LZO_LIBS= @LZO_LIBS@ LZO_INC= @LZO_INC@ +SD_LIBS = -lbacsd -lbaccfg -lbac -.SUFFIXES: .c .o + +.SUFFIXES: .c .o .lo .PHONY: .DONTCARE: @@ -107,22 +105,62 @@ LZO_INC= @LZO_INC@ .c.o: @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< + +.c.lo: + @echo "LT Compiling $<" + $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(TOKYOCABINET_INC) $(S3_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< + + #------------------------------------------------------------------------- -all: Makefile bacula-sd @STATIC_SD@ bls bextract bscan btape bcopy +all: Makefile libbacsd.la drivers bacula-sd @STATIC_SD@ \ + bls bextract bscan bcopy \ + bsdjson btape @echo "===== Make of stored is good ====" @echo " " -bacula-sd: Makefile $(SDOBJS) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) +bacula-sd: Makefile libbacsd.la $(SDOBJS) \ + ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) \ + ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) + @echo "Linking $@ ..." + $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L. -L../lib -o $@ $(SDOBJS) $(ZLIBS) \ + $(SD_LIBS) -lm $(DLIB) $(LIBS) $(WRAPLIBS) \ + $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) + +libbacsd.a: $(LIBBACSD_OBJS) + @echo "Making $@ ..." + $(AR) rc $@ $(LIBBACSD_OBJS) + $(RANLIB) $@ + +libbacsd.la: Makefile $(LIBBACSD_LOBJS) + @echo "Making $@ ..." + $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -o $@ \ + $(TOKYOCABINET_LIBS) $(LIBBACSD_LOBJS) \ + -export-dynamic -rpath $(libdir) -release $(LIBBACSD_LT_RELEASE) + +# +# Loadable driver +# +drivers: bacula-sd-cloud-driver.la bacula-sd-aligned-driver.la + +bacula-sd-cloud-driver.la: Makefile $(CLOUD_LOBJS) + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -shared $(CLOUD_LOBJS) -o $@ $(S3_LIBS) -rpath $(libdir) -module -export-dynamic -release $(LIBBACSD_LT_RELEASE) + +bacula-sd-aligned-driver.la: Makefile $(ALIGNED_LOBJS) + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -shared $(ALIGNED_LOBJS) -o $@ -rpath $(plugindir) \ + -module -export-dynamic -release $(LIBBACSD_LT_RELEASE) + + +bsdjson: Makefile $(JSONOBJS) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @echo "Linking $@ ..." - $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -o $@ $(SDOBJS) $(ZLIBS) \ + $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -o $@ $(JSONOBJS) $(ZLIBS) \ -lbaccfg -lbac -lm $(DLIB) $(LIBS) $(WRAPLIBS) \ - $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) + $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) static-bacula-sd: Makefile $(SDOBJS) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -static -L../lib -o $@ $(SDOBJS) $(ZLIBS) \ - -lbaccfg -lbac -lm $(DLIB) $(LIBS) $(WRAPLIBS) \ - $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) + $(SD_LIBS) -lm $(DLIB) $(LIBS) $(WRAPLIBS) \ + $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) strip $@ btape.o: btape.c @@ -130,60 +168,90 @@ btape.o: btape.c $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ -I$(basedir) $(DINCLUDE) $(CFLAGS) $< -btape: Makefile $(TAPEOBJS) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) +btape: Makefile $(TAPEOBJS) libbacsd.la drivers ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -o $@ $(TAPEOBJS) \ - -lbaccfg -lbac $(DLIB) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + $(SD_LIBS) $(DLIB) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + + +cloud_test.o: cloud_test.c + @echo "Compiling $<" + $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ + -I$(basedir) $(DINCLUDE) $(CFLAGS) $< + +cloud_test: Makefile cloud_test.o ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) $(BLSOBJS) libbacsd.la drivers + $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(CLOUDTESTOBJS) $(DLIB) \ + $(SD_LIBS) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bls.o: bls.c @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ -I$(basedir) $(DINCLUDE) $(CFLAGS) $< -bls: Makefile $(BLSOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) +bls: Makefile $(BLSOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @echo "Compiling $<" $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BLSOBJS) $(DLIB) \ - -lbacfind -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + $(SD_LIBS) -lbacfind -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bextract.o: bextract.c @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ -I$(basedir) $(DINCLUDE) $(CFLAGS) $(LZO_INC) $< -bextract: Makefile $(BEXTOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) +bextract: Makefile $(BEXTOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @echo "Compiling $<" $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZO_LIBS) \ - -lbacfind -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + $(SD_LIBS) -lbacfind -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bscan.o: bscan.c @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ -I$(basedir) $(DINCLUDE) $(CFLAGS) $< -bscan: Makefile $(SCNOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) \ +bscan: Makefile $(SCNOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) \ ../cats/libbacsql$(DEFAULT_ARCHIVE_TYPE) ../cats/libbaccats$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../cats -L../findlib -o $@ $(SCNOBJS) \ - -lbacsql -lbaccats $(DB_LIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + $(SD_LIBS) -lbacsql -lbaccats $(DB_LIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bcopy.o: bcopy.c @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ -I$(basedir) $(DINCLUDE) $(CFLAGS) $< -bcopy: Makefile $(COPYOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) +bcopy: Makefile $(COPYOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -o $@ $(COPYOBJS) \ - -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + $(SD_LIBS) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + +cloud_parts_test: Makefile cloud_parts.c + $(RMF) cloud_parts.o + $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) cloud_parts.c + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L../lib -o $@ cloud_parts.o $(DLIB) -lbac -lm $(LIBS) $(OPENSSL_LIBS) + rm -f cloud_parts.o + $(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) cloud_parts.c + Makefile: $(srcdir)/Makefile.in $(topdir)/config.status cd $(topdir) \ && CONFIG_FILES=$(thisdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status -install: all +libtool-install: all + $(MKDIR) $(DESTDIR)$(libdir) + $(RMF) $(DESTDIR)$(libdir)/libbacsd-*.so $(DESTDIR)$(libdir)/libbacsd.la + $(LIBTOOL_INSTALL_FINISH) $(INSTALL_LIB) libbacsd.la $(DESTDIR)$(libdir) + +libtool-uninstall: + $(LIBTOOL_UNINSTALL) $(RMF) $(DESTDIR)$(libdir)/libbacsd.la + +install: all @LIBTOOL_INSTALL_TARGET@ $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-sd $(DESTDIR)$(sbindir)/bacula-sd + $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bsdjson $(DESTDIR)$(sbindir)/bsdjson $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bls $(DESTDIR)$(sbindir)/bls $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bextract $(DESTDIR)$(sbindir)/bextract $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bcopy $(DESTDIR)$(sbindir)/bcopy $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bscan $(DESTDIR)$(sbindir)/bscan $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) btape $(DESTDIR)$(sbindir)/btape + @if test -f static-bacula-sd; then \ + $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) static-bacula-sd $(DESTDIR)$(sbindir)/static-bacula-sd; \ + fi @srcconf=bacula-sd.conf; \ if test -f ${DESTDIR}${sysconfdir}/$$srcconf; then \ destconf=$$srcconf.new; \ @@ -196,12 +264,19 @@ install: all @if test "x${sd_group}" != "x" -a "x${DESTDIR}" = "x" ; then \ chgrp -f ${sd_group} ${DESTDIR}${sysconfdir}/$$destconf; \ fi - @if test -f static-bacula-sd; then \ - $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) static-bacula-sd $(DESTDIR)$(sbindir)/static-bacula-sd; \ - fi + +install-cloud: bacula-sd-cloud-driver.la + $(MKDIR) $(DESTDIR)$(plugindir) + $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-sd-cloud-driver$(DEFAULT_SHARED_OBJECT_TYPE) $(DESTDIR)$(plugindir) + $(RMF) $(DESTDIR)$(plugindir)/bacula-sd-cloud-driver.la + +install-aligned: bacula-sd-aligned-driver.la + $(MKDIR) $(DESTDIR)$(plugindir) + $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-sd-aligned-driver$(DEFAULT_SHARED_OBJECT_TYPE) $(DESTDIR)$(plugindir) + $(RMF) $(DESTDIR)$(plugindir)/bacula-sd-aligned-driver.la uninstall: - (cd $(DESTDIR)$(sbindir); $(RMF) bacula-sd) + (cd $(DESTDIR)$(sbindir); $(RMF) bacula-sd bsdjson) (cd $(DESTDIR)$(sbindir); $(RMF) bls) (cd $(DESTDIR)$(sbindir); $(RMF) bextract) (cd $(DESTDIR)$(sbindir); $(RMF) bcopy) @@ -210,11 +285,13 @@ uninstall: (cd $(DESTDIR)$(sysconfdir); $(RMF) bacula-sd.conf bacula-sd.conf.new) libtool-clean: + @find . -name '*.lo' -print | xargs $(LIBTOOL_CLEAN) $(RMF) @$(RMF) -r .libs _libs + @$(RMF) *.la clean: libtool-clean @$(RMF) bacula-sd stored bls bextract bpool btape shmfree core core.* a.out *.o *.bak *~ *.intpro *.extpro 1 2 3 - @$(RMF) bscan bcopy static-bacula-sd + @$(RMF) bscan bsdjson bcopy static-bacula-sd realclean: clean @$(RMF) tags bacula-sd.conf @@ -236,7 +313,7 @@ depend: @$(MV) Makefile Makefile.bak @$(SED) "/^# DO NOT DELETE:/,$$ d" Makefile.bak > Makefile @$(ECHO) "# DO NOT DELETE: nice dependency list follows" >> Makefile - @$(CXX) -S -M $(CPPFLAGS) $(XINC) -I$(srcdir) -I$(basedir) *.c >> Makefile + @$(CXX) -S -M $(CPPFLAGS) $(XINC) $(S3_INC) $(TOKYOCABINET_INC) -I$(srcdir) -I$(basedir) *.c >> Makefile @if test -f Makefile ; then \ $(RMF) Makefile.bak; \ else \ diff --git a/bacula/src/stored/acquire.c b/bacula/src/stored/acquire.c index c1229d257c..016f76b654 100644 --- a/bacula/src/stored/acquire.c +++ b/bacula/src/stored/acquire.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Routines to acquire and release a device for read/write * * Written by Kern Sibbald, August MMII - * */ #include "bacula.h" /* pull in global headers */ @@ -54,6 +53,7 @@ bool acquire_device_for_read(DCR *dcr) Enter(rdbglvl); dev = dcr->dev; + ASSERT2(!dev->adata, "Called with adata dev. Wrong!"); dev->Lock_read_acquire(); Dmsg2(rdbglvl, "dcr=%p dev=%p\n", dcr, dcr->dev); Dmsg2(rdbglvl, "MediaType dcr=%s dev=%s\n", dcr->media_type, dev->device->media_type); @@ -198,7 +198,7 @@ bool acquire_device_for_read(DCR *dcr) /* Volume info is always needed because of VolType */ Dmsg1(rdbglvl, "dir_get_volume_info vol=%s\n", dcr->VolumeName); - if (!dir_get_volume_info(dcr, GET_VOL_INFO_FOR_READ)) { + if (!dir_get_volume_info(dcr, dcr->VolumeName, GET_VOL_INFO_FOR_READ)) { Dmsg2(rdbglvl, "dir_get_vol_info failed for vol=%s: %s\n", dcr->VolumeName, jcr->errmsg); Jmsg1(jcr, M_WARNING, 0, "Read acquire: %s", jcr->errmsg); @@ -229,7 +229,7 @@ bool acquire_device_for_read(DCR *dcr) * If it is a tape, it checks the volume name */ Dmsg1(rdbglvl, "open vol=%s\n", dcr->VolumeName); - if (!dev->open(dcr, OPEN_READ_ONLY)) { + if (!dev->open_device(dcr, OPEN_READ_ONLY)) { if (!dev->poll) { Jmsg4(jcr, M_WARNING, 0, _("Read open %s device %s Volume \"%s\" failed: ERR=%s\n"), dev->print_type(), dev->print_name(), dcr->VolumeName, dev->bstrerror()); @@ -240,17 +240,17 @@ bool acquire_device_for_read(DCR *dcr) /* Read Volume Label */ Dmsg0(rdbglvl, "calling read-vol-label\n"); - vol_label_status = read_dev_volume_label(dcr); + vol_label_status = dev->read_dev_volume_label(dcr); switch (vol_label_status) { case VOL_OK: - Dmsg0(rdbglvl, "Got correct volume.\n"); + Dmsg1(rdbglvl, "Got correct volume. VOL_OK: %s\n", dcr->VolCatInfo.VolCatName); ok = true; dev->VolCatInfo = dcr->VolCatInfo; /* structure assignment */ break; /* got it */ case VOL_IO_ERROR: Dmsg0(rdbglvl, "IO Error\n"); /* - * Send error message generated by read_dev_volume_label() + * Send error message generated by dev->read_dev_volume_label() * only we really had a tape mounted. This supresses superfluous * error messages when nothing is mounted. */ @@ -259,7 +259,7 @@ bool acquire_device_for_read(DCR *dcr) } goto default_path; case VOL_TYPE_ERROR: - Jmsg(jcr, M_FATAL, 0, dev->errmsg); + Jmsg(jcr, M_FATAL, 0, "%s", jcr->errmsg); goto get_out; case VOL_NAME_ERROR: Dmsg3(rdbglvl, "Vol name=%s want=%s drv=%s.\n", dev->VolHdr.VolumeName, @@ -270,7 +270,7 @@ bool acquire_device_for_read(DCR *dcr) dev->set_unload(); /* force unload of unwanted tape */ if (!unload_autochanger(dcr, -1)) { /* at least free the device so we can re-open with correct volume */ - dev->close(); + dev->close(dcr); free_volume(dev); } dev->set_load(); @@ -285,7 +285,7 @@ default_path: * If the device requires mount, close it, so the device can be ejected. */ if (dev->requires_mount()) { - dev->close(); + dev->close(dcr); free_volume(dev); } @@ -309,7 +309,7 @@ default_path: /* Volume info is always needed because of VolType */ Dmsg1(150, "dir_get_volume_info vol=%s\n", dcr->VolumeName); - if (!dir_get_volume_info(dcr, GET_VOL_INFO_FOR_READ)) { + if (!dir_get_volume_info(dcr, dcr->VolumeName, GET_VOL_INFO_FOR_READ)) { Dmsg2(150, "dir_get_vol_info failed for vol=%s: %s\n", dcr->VolumeName, jcr->errmsg); Jmsg1(jcr, M_WARNING, 0, "Read acquire: %s", jcr->errmsg); @@ -337,7 +337,6 @@ default_path: get_out: dev->Lock(); - dcr->clear_reserved(); /* If failed and not writing plugin close device */ if (!ok && dev->num_writers == 0 && dev->num_reserved() == 0) { generate_plugin_event(jcr, bsdEventDeviceClose, dcr); @@ -376,12 +375,12 @@ DCR *acquire_device_for_append(DCR *dcr) bool have_vol = false; Enter(200); + dcr->set_ameta(); init_device_wait_timers(dcr); dev->Lock_acquire(); /* only one job at a time */ dev->Lock(); - Dmsg1(100, "acquire_append device is %s\n", dev->is_tape()?"tape": - (dev->is_dvd()?"DVD":"disk")); + Dmsg1(100, "acquire_append device is %s\n", dev->print_type()); /* * With the reservation system, this should not happen */ @@ -476,10 +475,11 @@ bool release_device(DCR *dcr) DEVICE *dev = dcr->dev; bool ok = true; char tbuf[100]; - int was_blocked = BST_NOT_BLOCKED; + int was_blocked; dev->Lock(); + was_blocked = BST_NOT_BLOCKED; if (!dev->is_blocked()) { block_device(dev, BST_RELEASING); } else { @@ -513,15 +513,17 @@ bool release_device(DCR *dcr) dev->num_writers--; Dmsg1(100, "There are %d writers in release_device\n", dev->num_writers); if (dev->is_labeled()) { - Dmsg2(200, "dir_create_jobmedia. Release vol=%s dev=%s\n", - dev->getVolCatName(), dev->print_name()); + if (!dev->at_weot()) { + Dmsg2(200, "dir_create_jobmedia. Release vol=%s dev=%s\n", + dev->getVolCatName(), dev->print_name()); + } if (!dev->at_weot() && !dir_create_jobmedia_record(dcr)) { Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"), dcr->getVolCatName(), jcr->Job); } /* If no more writers, and no errors, and wrote something, write an EOF */ if (!dev->num_writers && dev->can_write() && dev->block_num > 0) { - dev->weof(1); + dev->weof(dcr, 1); write_ansi_ibm_labels(dcr, ANSI_EOF_LABEL, dev->VolHdr.VolumeName); } if (!dev->at_weot()) { @@ -552,45 +554,24 @@ bool release_device(DCR *dcr) /* If no writers, close if file or !CAP_ALWAYS_OPEN */ if (dev->num_writers == 0 && (!dev->is_tape() || !dev->has_cap(CAP_ALWAYSOPEN))) { generate_plugin_event(jcr, bsdEventDeviceClose, dcr); - if (!dev->close()) { + if (!dev->close(dcr) && dev->errmsg[0]) { Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); } free_volume(dev); } unlock_volumes(); - /* Fire off Alert command and include any output */ - if (!job_canceled(jcr) && dcr->device->alert_command) { - POOLMEM *alert; - int status = 1; - BPIPE *bpipe; - char line[MAXSTRING]; - alert = get_pool_memory(PM_FNAME); - alert = edit_device_codes(dcr, alert, dcr->device->alert_command, ""); - /* Wait maximum 5 minutes */ - bpipe = open_bpipe(alert, 60 * 5, "r"); - if (bpipe) { - while (fgets(line, sizeof(line), bpipe->rfd)) { - Jmsg(jcr, M_ALERT, 0, _("Alert: %s"), line); - } - status = close_bpipe(bpipe); - } else { - status = errno; - } - if (status != 0) { - berrno be; - Jmsg(jcr, M_ALERT, 0, _("3997 Bad alert command: %s: ERR=%s.\n"), - alert, be.bstrerror(status)); - } + /* Do new tape alert code */ + dev->get_tape_alerts(dcr); + /* alert_callback is in tape_alert.c -- show only most recent (last) alert */ + dev->show_tape_alerts(dcr, list_long, list_last, alert_callback); - Dmsg1(400, "alert status=%d\n", status); - free_pool_memory(alert); - } pthread_cond_broadcast(&dev->wait_next_vol); Dmsg2(100, "JobId=%u broadcast wait_device_release at %s\n", (uint32_t)jcr->JobId, bstrftimes(tbuf, sizeof(tbuf), (utime_t)time(NULL))); pthread_cond_broadcast(&wait_device_release); + /* * If we are the thread that blocked the device, then unblock it */ @@ -602,6 +583,8 @@ bool release_device(DCR *dcr) dev->Unlock(); } + dev->end_of_job(dcr); + if (dcr->keep_dcr) { dev->detach_dcr_from_dev(dcr); } else { @@ -644,6 +627,8 @@ DCR *new_dcr(JCR *jcr, DCR *dcr, DEVICE *dev, bool writing) dcr = (DCR *)malloc(sizeof(DCR)); memset(dcr, 0, sizeof(DCR)); dcr->tid = pthread_self(); + dcr->uploads = New(alist(100, false)); + dcr->downloads = New(alist(100, false)); dcr->spool_fd = -1; } dcr->jcr = jcr; /* point back to jcr */ @@ -655,9 +640,9 @@ DCR *new_dcr(JCR *jcr, DCR *dcr, DEVICE *dev, bool writing) ASSERT2(!dcr->attached_to_dev, "DCR is attached. Wrong!"); /* Set device information, possibly change device */ if (dev) { - dcr->free_blocks(); - dcr->block = new_block(dev); - dcr->ameta_block = dcr->block; + ASSERT2(!dev->adata, "Called with adata dev. Wrong!"); + dev->free_dcr_blocks(dcr); + dev->new_dcr_blocks(dcr); if (dcr->rec) { free_record(dcr->rec); } @@ -719,6 +704,7 @@ void DEVICE::attach_dcr_to_dev(DCR *dcr) if (jcr) Dmsg1(500, "JobId=%u enter attach_dcr_to_dev\n", (uint32_t)jcr->JobId); /* ***FIXME*** return error if dev not initiated */ if (!dcr->attached_to_dev && initiated && jcr && jcr->getJobType() != JT_SYSTEM) { + ASSERT2(!adata, "Called on adata dev. Wrong!"); Dmsg4(200, "Attach Jid=%d dcr=%p size=%d dev=%s\n", (uint32_t)jcr->JobId, dcr, attached_dcrs->size(), print_name()); attached_dcrs->append(dcr); /* attach dcr to device */ @@ -739,6 +725,7 @@ void DEVICE::detach_dcr_from_dev(DCR *dcr) Lock_dcrs(); /* Detach this dcr only if attached */ if (dcr->attached_to_dev) { + ASSERT2(!adata, "Called with adata dev. Wrong!"); dcr->unreserve_device(true); Dmsg4(200, "Detach Jid=%d dcr=%p size=%d to dev=%s\n", (uint32_t)dcr->jcr->JobId, dcr, attached_dcrs->size(), print_name()); @@ -749,8 +736,8 @@ void DEVICE::detach_dcr_from_dev(DCR *dcr) } /* Check if someone accidentally left a drive reserved, and clear it */ if (attached_dcrs->size() == 0 && num_reserved() > 0) { - Pmsg2(000, "Warning!!! dcrs=0 reserved=%d setting reserved==0. dev=%s\n", - num_reserved(), print_name()); + Pmsg3(000, "Warning!!! Detach %s DCR: dcrs=0 reserved=%d setting reserved==0. dev=%s\n", + dcr->is_writing() ? "writing" : "reading", num_reserved(), print_name()); m_num_reserved = 0; } dcr->attached_to_dev = false; @@ -772,7 +759,12 @@ void free_dcr(DCR *dcr) dcr->dev->detach_dcr_from_dev(dcr); } - dcr->free_blocks(); + if (dcr->dev) { + dcr->dev->free_dcr_blocks(dcr); + } else { + dcr->ameta_block = NULL; + free_block(dcr->block); + } if (dcr->rec) { free_record(dcr->rec); } @@ -782,6 +774,8 @@ void free_dcr(DCR *dcr) if (jcr && jcr->read_dcr == dcr) { jcr->read_dcr = NULL; } + delete dcr->uploads; + delete dcr->downloads; free(dcr); } @@ -796,4 +790,5 @@ static void set_dcr_from_vol(DCR *dcr, VOL_LIST *vol) bstrncpy(dcr->media_type, vol->MediaType, sizeof(dcr->media_type)); dcr->VolCatInfo.Slot = vol->Slot; dcr->VolCatInfo.InChanger = vol->Slot > 0; + dcr->CurrentVol = vol; /* Keep VOL_LIST pointer (freed at the end of the job) */ } diff --git a/bacula/src/stored/aligned_dev.c b/bacula/src/stored/aligned_dev.c new file mode 100644 index 0000000000..038a4e38d6 --- /dev/null +++ b/bacula/src/stored/aligned_dev.c @@ -0,0 +1,21 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Written by: Kern Sibbald, March MMXIII + */ diff --git a/bacula/src/stored/aligned_dev.h b/bacula/src/stored/aligned_dev.h new file mode 100644 index 0000000000..612b55b5d1 --- /dev/null +++ b/bacula/src/stored/aligned_dev.h @@ -0,0 +1,143 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Inspired by vtape.h + */ + +#ifndef _ALIGNED_DEV_H_ +#define _ALIGNED_DEV_H_ + +inline const char *DEVICE::aligned_name() const { return adev_name; } + +class aligned_dev : public file_dev { +public: + + aligned_dev(); + ~aligned_dev(); + + boffset_t get_adata_size(DCR *dcr); + boffset_t align_adata_addr(DCR *dcr, boffset_t addr); + boffset_t get_adata_addr(DCR *dcr); + void set_adata_addr(DCR *dcr); + void clear_adata_addr(); + + /* DEVICE virtual functions that we redefine */ + void setVolCatName(const char *name); + void setVolCatStatus(const char *status); + void free_dcr_blocks(DCR *dcr); + void new_dcr_blocks(DCR *dcr); + void updateVolCatBytes(uint64_t); + void updateVolCatBlocks(uint32_t); + void updateVolCatWrites(uint32_t); + void updateVolCatReads(uint32_t); + void updateVolCatReadBytes(uint64_t); + void updateVolCatPadding(uint64_t); + bool setVolCatAdataBytes(uint64_t bytes); + void updateVolCatHoleBytes(uint64_t bytes); + void device_specific_open(DCR *dcr); + void set_volcatinfo_from_dcr(DCR *dcr); + bool allow_maxbytes_concurrency(DCR *dcr); + bool flush_before_eos(DCR *dcr); + void set_nospace(); + void set_append(); + void set_read(); + void clear_nospace(); + void clear_append(); + void clear_read(); + void device_specific_init(JCR *jcr, DEVRES *device); + int d_close(int fd); + int d_open(const char *pathname, int flags); + int d_ioctl(int fd, ioctl_req_t request, char *mt_com); + ssize_t d_read(int fd, void *buffer, size_t count); + ssize_t d_write(int, const void *buffer, size_t count); + boffset_t lseek(DCR *dcr, off_t offset, int whence); + bool rewind(DCR *dcr); + bool reposition(DCR *dcr, uint64_t raddr); + bool open_device(DCR *dcr, int omode); + bool truncate(DCR *dcr); + bool close(DCR *dcr); + void term(DCR *dcr); + bool eod(DCR *dcr); + bool update_pos(DCR *dcr); + bool mount_file(int mount, int dotimeout); + bool is_indexed() { return !adata; }; + int read_dev_volume_label(DCR *dcr); + const char *print_type(); + DEVICE *get_dev(DCR *dcr); + uint32_t get_hi_addr(); + uint32_t get_low_addr(); + uint64_t get_full_addr(); + uint64_t get_full_addr(boffset_t addr); + bool do_size_checks(DCR *dcr, DEV_BLOCK *block); + bool write_volume_label_to_block(DCR *dcr); + bool write_volume_label_to_dev(DCR *dcr, + const char *VolName, const char *PoolName, + bool relabel, bool no_prelabel); + bool write_adata_label(DCR *dcr, DEV_RECORD *rec); + void write_adata(DCR *dcr, DEV_RECORD *rec); + void write_cont_adata(DCR *dcr, DEV_RECORD *rec); + int write_adata_rechdr(DCR *dcr, DEV_RECORD *rec); + bool read_adata_record_header(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec); + void read_adata_block_header(DCR *dcr); + int read_adata(DCR *dcr, DEV_RECORD *rec); + bool have_adata_header(DCR *dcr, DEV_RECORD *rec, int32_t FileIndex, + int32_t Stream, uint32_t VolSessionId); + void select_data_stream(DCR *dcr, DEV_RECORD *rec); + bool flush_block(DCR *dcr); + bool do_pre_write_checks(DCR *dcr, DEV_RECORD *rec); + + + + /* + * Locking and blocking calls + */ +#ifdef DEV_DEBUG_LOCK + void dbg_Lock(const char *, int); + void dbg_Unlock(const char *, int); + void dbg_rLock(const char *, int, bool locked=false); + void dbg_rUnlock(const char *, int); +#else + void Lock(); + void Unlock(); + void rLock(bool locked=false); + void rUnlock(); +#endif + +#ifdef SD_DEBUG_LOCK + void dbg_Lock_acquire(const char *, int); + void dbg_Unlock_acquire(const char *, int); + void dbg_Lock_read_acquire(const char *, int); + void dbg_Unlock_read_acquire(const char *, int); + void dbg_Lock_VolCatInfo(const char *, int); + void dbg_Unlock_VolCatInfo(const char *, int); +#else + void Lock_acquire(); + void Unlock_acquire(); + void Lock_read_acquire(); + void Unlock_read_acquire(); + void Lock_VolCatInfo(); + void Unlock_VolCatInfo(); +#endif + + void dblock(int why); /* in lock.c */ + void dunblock(bool locked=false); + +}; + +#endif /* _ALIGNED_DEV_H_ */ diff --git a/bacula/src/stored/aligned_read.c b/bacula/src/stored/aligned_read.c new file mode 100644 index 0000000000..6798657cf5 --- /dev/null +++ b/bacula/src/stored/aligned_read.c @@ -0,0 +1,25 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * record_read.c -- Volume (tape/disk) record read functions + * + * Kern Sibbald, April MMI + * added BB02 format October MMII + */ diff --git a/bacula/src/stored/aligned_write.c b/bacula/src/stored/aligned_write.c new file mode 100644 index 0000000000..1d73e7b4e9 --- /dev/null +++ b/bacula/src/stored/aligned_write.c @@ -0,0 +1,26 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * record_write.c -- Volume (tape/disk) record write functions + * + * Kern Sibbald, April MMI + * added BB02 format October MMII + * added aligned format November MMXII + */ diff --git a/bacula/src/stored/ansi_label.c b/bacula/src/stored/ansi_label.c index 8bbc91726b..2c81ceebb1 100644 --- a/bacula/src/stored/ansi_label.c +++ b/bacula/src/stored/ansi_label.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -22,7 +22,6 @@ * tape labels. * * Kern Sibbald, MMV - * */ #include "bacula.h" /* pull in global headers */ @@ -389,14 +388,14 @@ bool write_ansi_ibm_labels(DCR *dcr, int type, const char *VolName) be.bstrerror()); return false; } - dev->weof(1); + dev->weof(NULL, 1); return true; } else { Jmsg(jcr, M_FATAL, 0, _("Could not write ANSI HDR1 label.\n")); return false; } } - if (!dev->weof(1)) { + if (!dev->weof(NULL, 1)) { Jmsg(jcr, M_FATAL, 0, _("Error writing EOF to tape. ERR=%s"), dev->errmsg); return false; } diff --git a/bacula/src/stored/append.c b/bacula/src/stored/append.c index ad8d28ff8a..aed73b3cbf 100644 --- a/bacula/src/stored/append.c +++ b/bacula/src/stored/append.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -19,7 +19,6 @@ /* * Append code for Storage daemon * Kern Sibbald, May MM - * */ #include "bacula.h" @@ -52,6 +51,7 @@ void possible_incomplete_job(JCR *jcr, int32_t last_file_index) jcr->setJobStatus(JS_Incomplete); } } + /* * Append Data sent from Client (FD/SD) * @@ -99,6 +99,7 @@ bool do_append_data(JCR *jcr) return false; } + dev->start_of_job(dcr); jcr->sendJobStatus(JS_Running); //ASSERT(dev->VolCatInfo.VolCatName[0]); @@ -238,6 +239,10 @@ fi_checked: rec.data_len = qfd->msglen; rec.data = qfd->msg; /* use message buffer */ + /* Debug code: check if we must hangup or blowup */ + if (handle_hangup_blowup(jcr, jcr->JobFiles, jcr->JobBytes)) { + return false; + } Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n", rec.FileIndex, rec.VolSessionId, stream_to_ascii(buf1, rec.Stream,rec.FileIndex), @@ -271,7 +276,7 @@ fi_checked: } } - qfd->wait_read_sock(); + qfd->wait_read_sock((ok == false) || jcr->is_job_canceled()); free_GetMsg(qfd); if (eblock != NULL) { @@ -295,7 +300,19 @@ fi_checked: * Check if we can still write. This may not be the case * if we are at the end of the tape or we got a fatal I/O error. */ + dcr->set_ameta(); if (ok || dev->can_write()) { + if (!dev->flush_before_eos(dcr)) { + /* Print only if ok and not cancelled to avoid spurious messages */ + if (!jcr->is_job_canceled()) { + Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), + dev->print_name(), dev->bstrerror()); + Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n")); + possible_incomplete_job(jcr, last_file_index); + } + jcr->setJobStatus(JS_ErrorTerminated); + ok = false; + } if (!write_session_label(dcr, EOS_LABEL)) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { @@ -307,6 +324,8 @@ fi_checked: ok = false; } /* Flush out final partial block of this session */ + Dmsg1(200, "=== Flush adata=%d last block.\n", dcr->block->adata); + ASSERT(!dcr->block->adata); if (!dcr->write_final_block_to_device()) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { diff --git a/bacula/src/stored/askdir.c b/bacula/src/stored/askdir.c index 84e4c47c13..136f536737 100644 --- a/bacula/src/stored/askdir.c +++ b/bacula/src/stored/askdir.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +21,6 @@ * Reqests/commands from the Director are handled in dircmd.c * * Kern Sibbald, December 2000 - * */ #include "bacula.h" /* pull in global headers */ @@ -30,16 +29,17 @@ static const int dbglvl = 200; /* Requests sent to the Director */ -static char Find_media[] = "CatReq Job=%s FindMedia=%d pool_name=%s media_type=%s vol_type=%d\n"; -static char Get_Vol_Info[] = "CatReq Job=%s GetVolInfo VolName=%s write=%d\n"; -static char Update_media[] = "CatReq Job=%s UpdateMedia VolName=%s" +static char Find_media[] = "CatReq JobId=%ld FindMedia=%d pool_name=%s media_type=%s vol_type=%d\n"; +static char Get_Vol_Info[] = "CatReq JobId=%ld GetVolInfo VolName=%s write=%d\n"; +static char Update_media[] = "CatReq JobId=%ld UpdateMedia VolName=%s" " VolJobs=%u VolFiles=%u VolBlocks=%u VolBytes=%s VolABytes=%s" " VolHoleBytes=%s VolHoles=%u VolMounts=%u" " VolErrors=%u VolWrites=%u MaxVolBytes=%s EndTime=%s VolStatus=%s" " Slot=%d relabel=%d InChanger=%d VolReadTime=%s VolWriteTime=%s" - " VolFirstWritten=%s VolType=%u\n"; -static char Create_jobmedia[] = "CatReq Job=%s CreateJobMedia\n"; -static char FileAttributes[] = "UpdCat Job=%s FileAttributes "; + " VolFirstWritten=%s VolType=%u VolParts=%d VolCloudParts=%d" + " LastPartBytes=%lld Enabled=%d\n"; +static char Create_jobmedia[] = "CatReq JobId=%ld CreateJobMedia\n"; +static char FileAttributes[] = "UpdCat JobId=%ld FileAttributes "; /* Responses received from the Director */ static char OK_media[] = "1000 OK VolName=%127s VolJobs=%u VolFiles=%lu" @@ -49,7 +49,8 @@ static char OK_media[] = "1000 OK VolName=%127s VolJobs=%u VolFiles=%lu" " MaxVolBytes=%lld VolCapacityBytes=%lld VolStatus=%20s" " Slot=%ld MaxVolJobs=%lu MaxVolFiles=%lu InChanger=%ld" " VolReadTime=%lld VolWriteTime=%lld EndFile=%lu EndBlock=%lu" - " VolType=%lu LabelType=%ld MediaId=%lld ScratchPoolId=%lld\n"; + " VolType=%lu LabelType=%ld MediaId=%lld ScratchPoolId=%lld" + " VolParts=%d VolCloudParts=%d LastPartBytes=%lld Enabled=%d\n"; static char OK_create[] = "1000 OK CreateJobMedia\n"; @@ -58,7 +59,7 @@ static bthread_mutex_t vol_info_mutex = BTHREAD_MUTEX_PRIORITY(PRIO_SD_VOL_INFO) #ifdef needed -static char Device_update[] = "DevUpd Job=%s device=%s " +static char Device_update[] = "DevUpd JobId=%ld device=%s " "append=%d read=%d num_writers=%d " "open=%d labeled=%d offline=%d " "reserved=%d max_writers=%d " @@ -92,7 +93,7 @@ bool dir_update_device(JCR *jcr, DEVICE *dev) pm_strcpy(ChangerName, "*"); } ok = dir->fsend(Device_update, - jcr->Job, + jcr->JobId, dev_name.c_str(), dev->can_append()!=0, dev->can_read()!=0, dev->num_writers, @@ -120,7 +121,7 @@ bool dir_update_changer(JCR *jcr, AUTOCHANGER *changer) bash_spaces(MediaType); /* This is mostly to indicate that we are here */ ok = dir->fsend(Device_update, - jcr->Job, + jcr->JobId, dev_name.c_str(), /* Changer name */ 0, 0, 0, /* append, read, num_writers */ 0, 0, 0, /* is_open, is_labeled, offline */ @@ -138,11 +139,48 @@ bool dir_update_changer(JCR *jcr, AUTOCHANGER *changer) #endif +static AskDirHandler *askdir_handler = NULL; /* must be true when inside a "btools" */ + +/* + * btools must call this function, to modify behavior of some functions here + */ +AskDirHandler *init_askdir_handler(AskDirHandler *new_askdir_handler) +{ + AskDirHandler *old = askdir_handler; + askdir_handler = new_askdir_handler; + return old; +} + +/* + * Alternate function used by btools + */ +bool AskDirHandler::dir_ask_sysop_to_mount_volume(DCR *dcr, bool /*writing*/) +{ + DEVICE *dev = dcr->dev; + fprintf(stderr, _("Mount Volume \"%s\" on device %s and press return when ready: "), + dcr->VolumeName, dev->print_name()); + dev->close(dcr); + getchar(); + return true; +} + +bool AskDirHandler::dir_get_volume_info(DCR *dcr, const char *VolumeName, enum get_vol_info_rw writing) +{ + Dmsg0(100, "Fake dir_get_volume_info\n"); + dcr->setVolCatName(VolumeName); + Dmsg2(500, "Vol=%s VolType=%d\n", dcr->getVolCatName(), dcr->VolCatInfo.VolCatType); + return 1; +} + /** * Send current JobStatus to Director */ bool dir_send_job_status(JCR *jcr) { + if (askdir_handler) { + return askdir_handler->dir_send_job_status(jcr); + } + return jcr->sendJobStatus(); } @@ -166,6 +204,7 @@ static bool do_get_volume_info(DCR *dcr) BSOCK *dir = jcr->dir_bsock; VOLUME_CAT_INFO vol; int n; + int32_t Enabled; int32_t InChanger; dcr->setVolCatInfo(false); @@ -175,7 +214,6 @@ static bool do_get_volume_info(DCR *dcr) return false; } memset(&vol, 0, sizeof(vol)); - Dmsg1(dbglvl, "msg); n = sscanf(dir->msg, OK_media, vol.VolCatName, &vol.VolCatJobs, &vol.VolCatFiles, &vol.VolCatBlocks, &vol.VolCatAmetaBytes, @@ -186,8 +224,11 @@ static bool do_get_volume_info(DCR *dcr) &vol.Slot, &vol.VolCatMaxJobs, &vol.VolCatMaxFiles, &InChanger, &vol.VolReadTime, &vol.VolWriteTime, &vol.EndFile, &vol.EndBlock, &vol.VolCatType, - &vol.LabelType, &vol.VolMediaId, &vol.VolScratchPoolId); - if (n != 26) { + &vol.LabelType, &vol.VolMediaId, &vol.VolScratchPoolId, + &vol.VolCatParts, &vol.VolCatCloudParts, + &vol.VolLastPartBytes, &Enabled); + Dmsg2(dbglvl, "msg); + if (n != 30) { Dmsg1(dbglvl, "get_volume_info failed: ERR=%s", dir->msg); /* * Note, we can get an error here either because there is @@ -199,17 +240,19 @@ static bool do_get_volume_info(DCR *dcr) return false; } vol.InChanger = InChanger; /* bool in structure */ + vol.VolEnabled = Enabled; /* bool in structure */ vol.is_valid = true; - vol.VolCatBytes = vol.VolCatAmetaBytes; + vol.VolCatBytes = vol.VolCatAmetaBytes + vol.VolCatAdataBytes; unbash_spaces(vol.VolCatName); bstrncpy(dcr->VolumeName, vol.VolCatName, sizeof(dcr->VolumeName)); dcr->VolCatInfo = vol; /* structure assignment */ - Dmsg2(dbglvl, "do_reqest_vol_info return true slot=%d Volume=%s\n", - vol.Slot, vol.VolCatName); - Dmsg3(dbglvl, "Dir returned VolCatAmetaBytes=%lld Status=%s Vol=%s\n", - vol.VolCatAmetaBytes, - vol.VolCatStatus, vol.VolCatName); + Dmsg3(dbglvl, "do_reqest_vol_info return true slot=%d Volume=%s MediaId=%lld\n", + dcr->VolCatInfo.Slot, dcr->VolCatInfo.VolCatName, dcr->VolCatInfo.VolMediaId); + Dmsg5(dbglvl, "Dir returned VolCatAmetaBytes=%lld VolCatAdataBytes=%lld Status=%s Vol=%s MediaId=%lld\n", + dcr->VolCatInfo.VolCatAmetaBytes, dcr->VolCatInfo.VolCatAdataBytes, + dcr->VolCatInfo.VolCatStatus, dcr->VolCatInfo.VolCatName, + dcr->VolCatInfo.VolMediaId); return true; } @@ -224,15 +267,21 @@ static bool do_get_volume_info(DCR *dcr) * * Volume information returned in dcr->VolCatInfo */ -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) +bool dir_get_volume_info(DCR *dcr, + const char *VolumeName, + enum get_vol_info_rw writing) { + if (askdir_handler) { + return askdir_handler->dir_get_volume_info(dcr, VolumeName, writing); + } + JCR *jcr = dcr->jcr; BSOCK *dir = jcr->dir_bsock; P(vol_info_mutex); - dcr->setVolCatName(dcr->VolumeName); + dcr->setVolCatName(VolumeName); bash_spaces(dcr->getVolCatName()); - dir->fsend(Get_Vol_Info, jcr->Job, dcr->getVolCatName(), + dir->fsend(Get_Vol_Info, jcr->JobId, dcr->getVolCatName(), writing==GET_VOL_INFO_FOR_WRITE?1:0); Dmsg1(dbglvl, ">dird %s", dir->msg); unbash_spaces(dcr->getVolCatName()); @@ -257,6 +306,11 @@ bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) */ bool dir_find_next_appendable_volume(DCR *dcr) { + /* SD tools setup a handler because they have no connection to Dir */ + if (askdir_handler) { + return askdir_handler->dir_find_next_appendable_volume(dcr); + } + JCR *jcr = dcr->jcr; BSOCK *dir = jcr->dir_bsock; bool rtn; @@ -278,7 +332,7 @@ bool dir_find_next_appendable_volume(DCR *dcr) for (int vol_index=1; vol_index < 30; vol_index++) { bash_spaces(dcr->media_type); bash_spaces(dcr->pool_name); - dir->fsend(Find_media, jcr->Job, vol_index, dcr->pool_name, dcr->media_type, + dir->fsend(Find_media, jcr->JobId, vol_index, dcr->pool_name, dcr->media_type, dcr->dev->dev_type); unbash_spaces(dcr->media_type); unbash_spaces(dcr->pool_name); @@ -291,11 +345,17 @@ bool dir_find_next_appendable_volume(DCR *dcr) Dmsg1(dbglvl, "Got same vol = %s\n", lastVolume); break; } + /* If VolCatAdataBytes, we have ALIGNED_DEV */ + if (dcr->VolCatInfo.VolCatType == 0 && dcr->VolCatInfo.VolCatAdataBytes != 0) { + dcr->VolCatInfo.VolCatType = B_ALIGNED_DEV; + } /* - * If we have VolType and we are disk or ameta, the VolType must match + * If we have VolType and we are disk or aligned, the VolType must match */ + /* ***FIXME*** find better way to handle voltype */ if (dcr->VolCatInfo.VolCatType != 0 && - (dcr->dev->dev_type == B_FILE_DEV) && + (dcr->dev->dev_type == B_FILE_DEV || dcr->dev->dev_type == B_ALIGNED_DEV || + dcr->dev->dev_type == B_CLOUD_DEV) && dcr->dev->dev_type != (int)dcr->VolCatInfo.VolCatType) { Dmsg2(000, "Skip vol. Wanted VolType=%d Got=%d\n", dcr->dev->dev_type, dcr->VolCatInfo.VolCatType); continue; @@ -352,64 +412,82 @@ get_out: * back to the director. The information comes from the * dev record. */ -bool dir_update_volume_info(DCR *dcr, bool label, bool update_LastWritten) +bool dir_update_volume_info(DCR *dcr, bool label, bool update_LastWritten, + bool use_dcr) { + if (askdir_handler) { + return askdir_handler->dir_update_volume_info(dcr, label, update_LastWritten, use_dcr); + } + JCR *jcr = dcr->jcr; BSOCK *dir = jcr->dir_bsock; DEVICE *dev = dcr->ameta_dev; - VOLUME_CAT_INFO *vol; + VOLUME_CAT_INFO vol; char ed1[50], ed2[50], ed3[50], ed4[50], ed5[50], ed6[50], ed7[50], ed8[50]; int InChanger; bool ok = false; POOL_MEM VolumeName; - /* If system job, do not update catalog */ - if (jcr->getJobType() == JT_SYSTEM) { + /* If system job, do not update catalog, except if we explicitly force it. */ + if (jcr->getJobType() == JT_SYSTEM && + !dcr->force_update_volume_info) { return true; } - vol = &dev->VolCatInfo; - if (vol->VolCatName[0] == 0) { - Jmsg0(jcr, M_FATAL, 0, _("NULL Volume name. This shouldn't happen!!!\n")); - Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); - return false; - } - /* Lock during Volume update */ P(vol_info_mutex); dev->Lock_VolCatInfo(); - Dmsg3(100, "Update cat VolBytes=%lld Status=%s Vol=%s\n", - vol->VolCatAmetaBytes, vol->VolCatStatus, vol->VolCatName); + + if (use_dcr) { + vol = dcr->VolCatInfo; /* structure assignment */ + } else { + vol = dev->VolCatInfo; /* structure assignment */ + } + + /* This happens when nothing to update after fixup_device ... */ + if (vol.VolCatName[0] == 0) { + goto bail_out; + } + Dmsg4(100, "Update cat VolBytes=%lld VolABytes=%lld Status=%s Vol=%s\n", + vol.VolCatAmetaBytes, vol.VolCatAdataBytes, vol.VolCatStatus, vol.VolCatName); /* Just labeled or relabeled the tape */ if (label) { dev->setVolCatStatus("Append"); } // if (update_LastWritten) { - vol->VolLastWritten = time(NULL); + vol.VolLastWritten = time(NULL); // } - pm_strcpy(VolumeName, vol->VolCatName); + pm_strcpy(VolumeName, vol.VolCatName); bash_spaces(VolumeName); - InChanger = vol->InChanger; - vol->VolCatHoleBytes = 0; - + InChanger = vol.InChanger; + /* Insanity test */ + if (vol.VolCatHoleBytes > (((uint64_t)2)<<60)) { + Pmsg1(010, "VolCatHoleBytes too big: %lld. Reset to zero.\n", + vol.VolCatHoleBytes); + vol.VolCatHoleBytes = 0; + } /* Set device type where this Volume used */ - if (vol->VolCatType == 0) { - vol->VolCatType = dev->dev_type; - } - dir->fsend(Update_media, jcr->Job, - VolumeName.c_str(), vol->VolCatJobs, vol->VolCatFiles, - vol->VolCatBlocks, edit_uint64(vol->VolCatAmetaBytes, ed1), - edit_uint64(vol->VolCatAdataBytes, ed2), - edit_uint64(vol->VolCatHoleBytes, ed3), - vol->VolCatHoles, vol->VolCatMounts, vol->VolCatErrors, - vol->VolCatWrites, edit_uint64(vol->VolCatMaxBytes, ed4), - edit_uint64(vol->VolLastWritten, ed5), - vol->VolCatStatus, vol->Slot, label, + if (vol.VolCatType == 0) { + vol.VolCatType = dev->dev_type; + } + dir->fsend(Update_media, jcr->JobId, + VolumeName.c_str(), vol.VolCatJobs, vol.VolCatFiles, + vol.VolCatBlocks, edit_uint64(vol.VolCatAmetaBytes, ed1), + edit_uint64(vol.VolCatAdataBytes, ed2), + edit_uint64(vol.VolCatHoleBytes, ed3), + vol.VolCatHoles, vol.VolCatMounts, vol.VolCatErrors, + vol.VolCatWrites, edit_uint64(vol.VolCatMaxBytes, ed4), + edit_uint64(vol.VolLastWritten, ed5), + vol.VolCatStatus, vol.Slot, label, InChanger, /* bool in structure */ - edit_int64(vol->VolReadTime, ed6), - edit_int64(vol->VolWriteTime, ed7), - edit_uint64(vol->VolFirstWritten, ed8), - vol->VolCatType); + edit_int64(vol.VolReadTime, ed6), + edit_int64(vol.VolWriteTime, ed7), + edit_uint64(vol.VolFirstWritten, ed8), + vol.VolCatType, + vol.VolCatParts, + vol.VolCatCloudParts, + vol.VolLastPartBytes, + vol.VolEnabled); Dmsg1(100, ">dird %s", dir->msg); /* Do not lock device here because it may be locked from label */ @@ -422,29 +500,31 @@ bool dir_update_volume_info(DCR *dcr, bool label, bool update_LastWritten) if (!do_get_volume_info(dcr)) { Jmsg(jcr, M_FATAL, 0, "%s", jcr->errmsg); Dmsg2(dbglvl, _("Didn't get vol info vol=%s: ERR=%s"), - vol->VolCatName, jcr->errmsg); + vol.VolCatName, jcr->errmsg); goto bail_out; } Dmsg1(100, "get_volume_info() %s", dir->msg); + /* Update dev Volume info in case something changed (e.g. expired) */ - vol->Slot = dev->VolCatInfo.Slot; - bstrncpy(vol->VolCatStatus, dcr->VolCatInfo.VolCatStatus, sizeof(vol->VolCatStatus)); - - dcr->VolCatInfo.VolCatAdataBytes = dev->VolCatInfo.VolCatAdataBytes; - dcr->VolCatInfo.VolCatAmetaBytes = dev->VolCatInfo.VolCatAmetaBytes; - dcr->VolCatInfo.VolCatHoleBytes = dev->VolCatInfo.VolCatHoleBytes; - dcr->VolCatInfo.VolCatHoles = dev->VolCatInfo.VolCatHoles; - dcr->VolCatInfo.VolCatPadding = dev->VolCatInfo.VolCatPadding; - dcr->VolCatInfo.VolCatAmetaPadding = dev->VolCatInfo.VolCatAmetaPadding; - dcr->VolCatInfo.VolCatAdataPadding = dev->VolCatInfo.VolCatAdataPadding; - dcr->VolCatInfo.VolCatFiles = dev->VolCatInfo.VolCatFiles; - dcr->VolCatInfo.VolCatBytes = dev->VolCatInfo.VolCatBytes; - dcr->VolCatInfo.VolCatMounts = dev->VolCatInfo.VolCatMounts; - dcr->VolCatInfo.VolCatJobs = dev->VolCatInfo.VolCatJobs; - dcr->VolCatInfo.VolCatFiles = dev->VolCatInfo.VolCatFiles; - dcr->VolCatInfo.VolCatRecycles = dev->VolCatInfo.VolCatRecycles; - dcr->VolCatInfo.VolCatWrites = dev->VolCatInfo.VolCatWrites; - dcr->VolCatInfo.VolCatReads = dev->VolCatInfo.VolCatReads; + if (!use_dcr) { + dcr->VolCatInfo.Slot = dev->VolCatInfo.Slot; + bstrncpy(dcr->VolCatInfo.VolCatStatus, dev->VolCatInfo.VolCatStatus, sizeof(vol.VolCatStatus)); + dcr->VolCatInfo.VolCatAdataBytes = dev->VolCatInfo.VolCatAdataBytes; + dcr->VolCatInfo.VolCatAmetaBytes = dev->VolCatInfo.VolCatAmetaBytes; + dcr->VolCatInfo.VolCatHoleBytes = dev->VolCatInfo.VolCatHoleBytes; + dcr->VolCatInfo.VolCatHoles = dev->VolCatInfo.VolCatHoles; + dcr->VolCatInfo.VolCatPadding = dev->VolCatInfo.VolCatPadding; + dcr->VolCatInfo.VolCatAmetaPadding = dev->VolCatInfo.VolCatAmetaPadding; + dcr->VolCatInfo.VolCatAdataPadding = dev->VolCatInfo.VolCatAdataPadding; + dcr->VolCatInfo.VolCatFiles = dev->VolCatInfo.VolCatFiles; + dcr->VolCatInfo.VolCatBytes = dev->VolCatInfo.VolCatBytes; + dcr->VolCatInfo.VolCatMounts = dev->VolCatInfo.VolCatMounts; + dcr->VolCatInfo.VolCatJobs = dev->VolCatInfo.VolCatJobs; + dcr->VolCatInfo.VolCatFiles = dev->VolCatInfo.VolCatFiles; + dcr->VolCatInfo.VolCatRecycles = dev->VolCatInfo.VolCatRecycles; + dcr->VolCatInfo.VolCatWrites = dev->VolCatInfo.VolCatWrites; + dcr->VolCatInfo.VolCatReads = dev->VolCatInfo.VolCatReads; + } ok = true; } @@ -456,13 +536,15 @@ bail_out: struct JOBMEDIA_ITEM { dlink link; + int64_t VolMediaId; + uint64_t StartAddr; + uint64_t EndAddr; uint32_t VolFirstIndex; uint32_t VolLastIndex; uint32_t StartFile; uint32_t EndFile; uint32_t StartBlock; uint32_t EndBlock; - int64_t VolMediaId; }; void create_jobmedia_queue(JCR *jcr) @@ -473,17 +555,20 @@ void create_jobmedia_queue(JCR *jcr) bool flush_jobmedia_queue(JCR *jcr) { + if (askdir_handler) { + return askdir_handler->flush_jobmedia_queue(jcr); + } + JOBMEDIA_ITEM *item; BSOCK *dir = jcr->dir_bsock; bool ok; if (!jcr->jobmedia_queue || jcr->jobmedia_queue->size() == 0) { - //Dmsg0(000, "No jobmedia_queue\n"); return true; /* should never happen */ } Dmsg1(400, "=== Flush jobmedia queue = %d\n", jcr->jobmedia_queue->size()); - dir->fsend(Create_jobmedia, jcr->Job); + dir->fsend(Create_jobmedia, jcr->JobId); foreach_dlist(item, jcr->jobmedia_queue) { ok = dir->fsend("%u %u %u %u %u %u %lld\n", item->VolFirstIndex, item->VolLastIndex, @@ -510,11 +595,16 @@ bool flush_jobmedia_queue(JCR *jcr) return true; } + /* * After writing a Volume, create the JobMedia record. */ bool dir_create_jobmedia_record(DCR *dcr, bool zero) { + if (askdir_handler) { + return askdir_handler->dir_create_jobmedia_record(dcr, zero); + } + JCR *jcr = dcr->jcr; BSOCK *dir = jcr->dir_bsock; JOBMEDIA_ITEM *item; @@ -524,15 +614,16 @@ bool dir_create_jobmedia_record(DCR *dcr, bool zero) return true; } if (!zero && dcr->VolLastIndex == 0) { - Dmsg7(dbglvl, "JobMedia Vol=%s wrote=%d MediaId=%d FI=%d LI=%d StartBlock=%d EndBlock=%d Suppressed\n", + Pmsg7(0/*dbglvl*/, "Discard: JobMedia Vol=%s wrote=%d MediaId=%lld FI=%lu LI=%lu StartAddr=%lld EndAddr=%lld\n", dcr->VolumeName, dcr->WroteVol, dcr->VolMediaId, - dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartBlock, dcr->EndBlock); + dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartAddr, dcr->EndAddr); return true; /* nothing written to the Volume */ } - if (!zero && dcr->StartFile == dcr->EndFile && dcr->EndBlock < dcr->StartBlock) { - Dmsg7(dbglvl, "JobMedia Vol=%s wrote=%d MediaId=%d FI=%d LI=%d StartBlock=%d EndBlock=%d Suppressed\n", + /* Throw out records where the start address is bigger than the end */ + if (!zero && dcr->StartAddr > dcr->EndAddr) { + Pmsg7(0/*dbglvl*/, "Discard: JobMedia Vol=%s wrote=%d MediaId=%lld FI=%lu LI=%lu StartAddr=%lld EndAddr=%lld\n", dcr->VolumeName, dcr->WroteVol, dcr->VolMediaId, - dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartBlock, dcr->EndBlock); + dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartAddr, dcr->EndAddr); return true; } @@ -543,10 +634,10 @@ bool dir_create_jobmedia_record(DCR *dcr, bool zero) /* Throw out records where FI is zero -- i.e. nothing done */ if (!zero && dcr->VolFirstIndex == 0 && - (dcr->StartBlock != 0 || dcr->EndBlock != 0)) { - Dmsg7(dbglvl, "Discard: JobMedia Vol=%s wrote=%d MediaId=%d FI=%d LI=%d StartBlock=%d EndBlock=%d Suppressed\n", + (dcr->StartAddr != 0 || dcr->EndAddr != 0)) { + Pmsg7(0/*dbglvl*/, "Discard: JobMedia Vol=%s wrote=%d MediaId=%lld FI=%lu LI=%lu StartAddr=%lld EndAddr=%lld\n", dcr->VolumeName, dcr->WroteVol, dcr->VolMediaId, - dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartBlock, dcr->EndBlock); + dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartAddr, dcr->EndAddr); return true; } @@ -565,23 +656,25 @@ bool dir_create_jobmedia_record(DCR *dcr, bool zero) Dmsg1(100, "======= Set FI=%ld\n", dcr->VolLastIndex); } - Dmsg7(100, "JobMedia Vol=%s wrote=%d MediaId=%d FI=%d LI=%d StartBlock=%d EndBlock=%d Wrote\n", + Dmsg7(100, "Queue JobMedia Vol=%s wrote=%d MediaId=%lld FI=%lu LI=%lu StartAddr=%lld EndAddr=%lld\n", dcr->VolumeName, dcr->WroteVol, dcr->VolMediaId, - dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartBlock, dcr->EndBlock); - dcr->WroteVol = false; + dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartAddr, dcr->EndAddr); item = (JOBMEDIA_ITEM *)malloc(sizeof(JOBMEDIA_ITEM)); if (zero) { item->VolFirstIndex = item->VolLastIndex = 0; item->StartFile = item->EndFile = 0; item->StartBlock = item->EndBlock = 0; + item->StartAddr = item->EndAddr = 0; item->VolMediaId = dcr->VolMediaId; } else { item->VolFirstIndex = dcr->VolFirstIndex; item->VolLastIndex = dcr->VolLastIndex; - item->StartFile = dcr->StartFile; - item->EndFile = dcr->EndFile; - item->StartBlock = dcr->StartBlock; - item->EndBlock = dcr->EndBlock; + item->StartFile = (uint32_t)(dcr->StartAddr >> 32); + item->EndFile = (uint32_t)(dcr->EndAddr >> 32); + item->StartBlock = (uint32_t)dcr->StartAddr; + item->EndBlock = (uint32_t)dcr->EndAddr; + item->StartAddr = dcr->StartAddr; + item->EndAddr = dcr->EndAddr; item->VolMediaId = dcr->VolMediaId; } jcr->jobmedia_queue->append(item); @@ -591,13 +684,13 @@ bool dir_create_jobmedia_record(DCR *dcr, bool zero) } dcr->VolFirstIndex = dcr->VolLastIndex = 0; - dcr->StartFile = dcr->EndFile = 0; - dcr->StartBlock = dcr->EndBlock = 0; + dcr->StartAddr = dcr->EndAddr = 0; + dcr->VolMediaId = 0; + dcr->WroteVol = false; return ok; } - -/** +/* * Update File Attribute data * We do the following: * 1. expand the bsock buffer to be large enough @@ -615,6 +708,10 @@ bool dir_create_jobmedia_record(DCR *dcr, bool zero) */ bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { + if (askdir_handler) { + return askdir_handler->dir_update_file_attributes(dcr, rec); + } + JCR *jcr = dcr->jcr; BSOCK *dir = jcr->dir_bsock; ser_declare; @@ -626,7 +723,7 @@ bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) dir->msg = check_pool_memory_size(dir->msg, sizeof(FileAttributes) + MAX_NAME_LENGTH + sizeof(DEV_RECORD) + rec->data_len + 1); dir->msglen = bsnprintf(dir->msg, sizeof(FileAttributes) + - MAX_NAME_LENGTH + 1, FileAttributes, jcr->Job); + MAX_NAME_LENGTH + 1, FileAttributes, jcr->JobId); ser_begin(dir->msg + dir->msglen, 0); ser_uint32(rec->VolSessionId); ser_uint32(rec->VolSessionTime); @@ -665,6 +762,10 @@ bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) */ bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { + if (askdir_handler) { + return askdir_handler->dir_ask_sysop_to_create_appendable_volume(dcr); + } + int stat = W_TIMEOUT; DEVICE *dev = dcr->dev; JCR *jcr = dcr->jcr; @@ -737,13 +838,12 @@ bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) } get_out: - dev->poll = false; jcr->sendJobStatus(JS_Running); Dmsg0(dbglvl, "leave dir_ask_sysop_to_create_appendable_volume\n"); return true; } -/** +/* * Request to mount specific Volume * * Entered with device blocked and dcr->VolumeName is desired @@ -757,6 +857,10 @@ get_out: */ bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool write_access) { + if (askdir_handler) { + return askdir_handler->dir_ask_sysop_to_mount_volume(dcr, write_access); + } + int stat = W_TIMEOUT; DEVICE *dev = dcr->dev; JCR *jcr = dcr->jcr; @@ -774,18 +878,7 @@ bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool write_access) return false; } - for ( ;; ) { - if (job_canceled(jcr)) { - Mmsg(dev->errmsg, _("Job %s canceled while waiting for mount on Storage Device %s.\n"), - jcr->Job, dev->print_name()); - dev->poll = false; - return false; - } - - if (dev->is_dvd()) { - dev->unmount(0); - } - + for ( ; job_canceled(jcr); ) { /* * If we are not polling, and the wait timeout or the * user explicitly did a mount, send him the message. @@ -850,6 +943,13 @@ bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool write_access) } get_out: + if (job_canceled(jcr)) { + Mmsg(dev->errmsg, _("Job %s canceled while waiting for mount on Storage Device %s.\n"), + jcr->Job, dev->print_name()); + dev->poll = false; + return false; + } + jcr->sendJobStatus(JS_Running); Dmsg0(100, "leave dir_ask_sysop_to_mount_volume\n"); return true; diff --git a/bacula/src/stored/authenticate.c b/bacula/src/stored/authenticate.c index a1dc2fa192..832557579b 100644 --- a/bacula/src/stored/authenticate.c +++ b/bacula/src/stored/authenticate.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Authenticate caller * * Written by Kern Sibbald, October 2000 - * */ @@ -33,6 +32,14 @@ const int dbglvl = 50; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +/* Version at end of Hello + * prior to 06Aug13 no version + * 1 06Aug13 - added comm line compression + * 2 13Dec13 - added api version to status command + */ +#define SD_VERSION 2 + + /* * Authenticate the Director */ @@ -316,6 +323,13 @@ bool authenticate_storagedaemon(JCR *jcr) goto auth_fatal; } sscanf(sd->msg, "3000 OK Hello %d", &sd_version); + if (sd_version >= 1 && me->comm_compression) { + sd->set_compress(); + } else { + sd->clear_compress(); + Dmsg0(050, "*** No FD compression with SD\n"); + } + /* At this point, we have successfully connected */ auth_fatal: diff --git a/bacula/src/stored/autochanger.c b/bacula/src/stored/autochanger.c index e3d8180683..ca80e2d347 100644 --- a/bacula/src/stored/autochanger.c +++ b/bacula/src/stored/autochanger.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +21,6 @@ * Routines for handling the autochanger. * * Written by Kern Sibbald, August MMII - * */ #include "bacula.h" /* pull in global headers */ @@ -32,7 +31,7 @@ static const int dbglvl = 60; /* Forward referenced functions */ static void lock_changer(DCR *dcr); static void unlock_changer(DCR *dcr); -static bool unload_other_drive(DCR *dcr, int slot); +static bool unload_other_drive(DCR *dcr, int slot, bool writing); bool DCR::is_virtual_autochanger() { @@ -96,7 +95,7 @@ int autoload_device(DCR *dcr, bool writing, BSOCK *dir) { JCR *jcr = dcr->jcr; DEVICE * volatile dev = dcr->dev; - char *vol_name = dcr->VolumeName; + char *new_vol_name = dcr->VolumeName; int slot; int drive = dev->drive_index; int rtn_stat = -1; /* error status */ @@ -177,7 +176,7 @@ int autoload_device(DCR *dcr, bool writing, BSOCK *dir) } /* Make sure desired slot is unloaded */ - if (!unload_other_drive(dcr, slot)) { + if (!unload_other_drive(dcr, slot, writing)) { goto bail_out; } @@ -187,17 +186,22 @@ int autoload_device(DCR *dcr, bool writing, BSOCK *dir) lock_changer(dcr); Dmsg2(dbglvl, "Doing changer load slot %d %s\n", slot, dev->print_name()); Jmsg(jcr, M_INFO, 0, - _("3304 Issuing autochanger \"load slot %d, drive %d\" command for vol %s.\n"), - slot, drive, vol_name); + _("3304 Issuing autochanger \"load Volume %s, Slot %d, Drive %d\" command.\n"), + new_vol_name, slot, drive); + Dmsg3(dbglvl, + "3304 Issuing autochanger \"load Volume %s, Slot %d, Drive %d\" command.\n", + new_vol_name, slot, drive); + dcr->VolCatInfo.Slot = slot; /* slot to be loaded */ changer = edit_device_codes(dcr, changer, dcr->device->changer_command, "load"); - dev->close(); + dev->close(dcr); Dmsg1(dbglvl, "Run program=%s\n", changer); status = run_program_full_output(changer, timeout, results.addr()); if (status == 0) { - Jmsg(jcr, M_INFO, 0, _("3305 Autochanger \"load slot %d, drive %d\", status is OK for vol %s.\n"), - slot, drive, vol_name); - Dmsg3(dbglvl, "OK: load slot %d, drive %d vol %s.\n", slot, drive, vol_name); + Jmsg(jcr, M_INFO, 0, _("3305 Autochanger \"load Volume %s, Slot %d, Drive %d\", status is OK.\n"), + new_vol_name, slot, drive); + Dmsg3(dbglvl, "OK: load volume %s, slot %d, drive %d.\n", new_vol_name, slot, drive); + bstrncpy(dev->LoadedVolName, new_vol_name, sizeof(dev->LoadedVolName)); dev->set_slot(slot); /* set currently loaded slot */ if (dev->vol) { /* We just swapped this Volume so it cannot be swapping any more */ @@ -206,11 +210,12 @@ int autoload_device(DCR *dcr, bool writing, BSOCK *dir) } else { berrno be; be.set_errno(status); - Dmsg4(dbglvl, "Error: load slot %d, drive %d, bad stats=%s.\nResults=%s\n", slot, drive, + Dmsg5(dbglvl, "Error: load Volume %s, Slot %d, Drive %d, bad stats=%s.\nResults=%s\n", + new_vol_name, slot, drive, be.bstrerror(), results.c_str()); - Jmsg(jcr, M_FATAL, 0, _("3992 Bad autochanger \"load slot %d, drive %d\": " + Jmsg(jcr, M_FATAL, 0, _("3992 Bad autochanger \"load Volume %s Slot %d, Drive %d\": " "ERR=%s.\nResults=%s\n"), - slot, drive, be.bstrerror(), results.c_str()); + new_vol_name, slot, drive, be.bstrerror(), results.c_str()); rtn_stat = -1; /* hard error */ dev->clear_slot(); /* mark unknown */ } @@ -218,6 +223,7 @@ int autoload_device(DCR *dcr, bool writing, BSOCK *dir) } else { status = 0; /* we got what we want */ dev->set_slot(slot); /* set currently loaded slot */ + bstrncpy(dev->LoadedVolName, new_vol_name, sizeof(dev->LoadedVolName)); } Dmsg1(dbglvl, "After changer, status=%d\n", status); if (status == 0) { /* did we succeed? */ @@ -352,7 +358,7 @@ bool unload_autochanger(DCR *dcr, int loaded) { DEVICE *dev = dcr->dev; JCR *jcr = dcr->jcr; - const char *vol_name = dcr->VolumeName; + const char *old_vol_name; int slot; uint32_t timeout = dcr->device->max_changer_wait; bool ok = true; @@ -373,8 +379,10 @@ bool unload_autochanger(DCR *dcr, int loaded) } lock_changer(dcr); - if (!vol_name || !vol_name[0]) { - vol_name = "unknown"; + if (dev->LoadedVolName[0]) { + old_vol_name = dev->LoadedVolName; + } else { + old_vol_name = "*Unknown*"; } if (loaded < 0) { loaded = get_autochanger_loaded_slot(dcr); @@ -387,30 +395,34 @@ bool unload_autochanger(DCR *dcr, int loaded) POOL_MEM results(PM_MESSAGE); POOLMEM *changer = get_pool_memory(PM_FNAME); Jmsg(jcr, M_INFO, 0, - _("3307 Issuing autochanger \"unload slot %d, drive %d\" command for vol %s.\n"), - loaded, dev->drive_index, vol_name); + _("3307 Issuing autochanger \"unload Volume %s, Slot %d, Drive %d\" command.\n"), + old_vol_name, loaded, dev->drive_index); + Dmsg3(dbglvl, + "3307 Issuing autochanger \"unload Volume %s, Slot %d, Drive %d\" command.\n", + old_vol_name, loaded, dev->drive_index); slot = dcr->VolCatInfo.Slot; dcr->VolCatInfo.Slot = loaded; changer = edit_device_codes(dcr, changer, dcr->device->changer_command, "unload"); - dev->close(); + dev->close(dcr); Dmsg1(dbglvl, "Run program=%s\n", changer); int stat = run_program_full_output(changer, timeout, results.addr()); dcr->VolCatInfo.Slot = slot; if (stat != 0) { berrno be; be.set_errno(stat); - Jmsg(jcr, M_INFO, 0, _("3995 Bad autochanger \"unload slot %d, drive %d\": " + Jmsg(jcr, M_INFO, 0, _("3995 Bad autochanger \"unload Volume %s, Slot %d, Drive %d\": " "ERR=%s\nResults=%s\n"), - loaded, dev->drive_index, be.bstrerror(), results.c_str()); - Dmsg4(dbglvl, "Error: load slot %d, drive %d, bad stats=%s.\nResults=%s\n", - loaded, dev->drive_index, + old_vol_name, loaded, dev->drive_index, be.bstrerror(), results.c_str()); + Dmsg5(dbglvl, "Error: unload Volume %s, Slot %d, Drive %d, bad stats=%s.\nResults=%s\n", + old_vol_name, loaded, dev->drive_index, be.bstrerror(), results.c_str()); ok = false; dev->clear_slot(); /* unknown */ } else { dev->set_slot(0); /* unload is OK, mark nothing loaded */ dev->clear_unload(); + dev->LoadedVolName[0] = 0; /* clear loaded volume name */ } free_pool_memory(changer); } @@ -425,7 +437,7 @@ bool unload_autochanger(DCR *dcr, int loaded) /* * Unload the slot if mounted in a different drive */ -static bool unload_other_drive(DCR *dcr, int slot) +static bool unload_other_drive(DCR *dcr, int slot, bool writing) { DEVICE *dev = NULL; DEVICE *dev_save; @@ -488,28 +500,37 @@ static bool unload_other_drive(DCR *dcr, int slot) Dmsg3(dbglvl, "Slot=%d drive=%d found in dev=%s\n", slot, dev->drive_index, dev->print_name()); } - /* The Volume we want is on another device. */ - if (dev->is_busy()) { - Dmsg4(dbglvl, "Vol %s for dev=%s in use dev=%s slot=%d\n", - dcr->VolumeName, dcr->dev->print_name(), - dev->print_name(), slot); - } - for (int i=0; i < 3; i++) { + /* + * The Volume we want is on another device. + * If we want the Volume to read it, and the other device where the + * Volume is currently is not open, we simply unload the Volume then + * then the subsequent code will load it in the desired drive. + * If want to write or the device is open, we attempt to wait for + * the Volume to become available. + */ + if (writing || dev->is_open()) { if (dev->is_busy()) { - Dmsg0(40, "Device is busy. Calling wait_for_device()\n"); - wait_for_device(dcr, retries); - continue; + Dmsg4(dbglvl, "Vol %s for dev=%s in use dev=%s slot=%d\n", + dcr->VolumeName, dcr->dev->print_name(), + dev->print_name(), slot); + } + for (int i=0; i < 3; i++) { + if (dev->is_busy()) { + Dmsg0(40, "Device is busy. Calling wait_for_device()\n"); + wait_for_device(dcr, retries); + continue; + } + break; + } + if (dev->is_busy()) { + Jmsg(dcr->jcr, M_WARNING, 0, _("Volume \"%s\" wanted on %s is in use by device %s\n"), + dcr->VolumeName, dcr->dev->print_name(), dev->print_name()); + Dmsg4(dbglvl, "Vol %s for dev=%s is busy dev=%s slot=%d\n", + dcr->VolumeName, dcr->dev->print_name(), dev->print_name(), dev->get_slot()); + Dmsg2(dbglvl, "num_writ=%d reserv=%d\n", dev->num_writers, dev->num_reserved()); + volume_unused(dcr); + return false; } - break; - } - if (dev->is_busy()) { - Jmsg(dcr->jcr, M_WARNING, 0, _("Volume \"%s\" wanted on %s is in use by device %s\n"), - dcr->VolumeName, dcr->dev->print_name(), dev->print_name()); - Dmsg4(dbglvl, "Vol %s for dev=%s is busy dev=%s slot=%d\n", - dcr->VolumeName, dcr->dev->print_name(), dev->print_name(), dev->get_slot()); - Dmsg2(dbglvl, "num_writ=%d reserv=%d\n", dev->num_writers, dev->num_reserved()); - volume_unused(dcr); - return false; } return unload_dev(dcr, dev); } @@ -523,7 +544,7 @@ bool unload_dev(DCR *dcr, DEVICE *dev) bool ok = true; uint32_t timeout = dcr->device->max_changer_wait; AUTOCHANGER *changer = dcr->dev->device->changer_res; - const char *vol_name = dcr->VolumeName; + const char *old_vol_name = dcr->VolumeName; DEVICE *save_dev; int save_slot; @@ -549,23 +570,25 @@ bool unload_dev(DCR *dcr, DEVICE *dev) save_slot = dcr->VolCatInfo.Slot; dcr->VolCatInfo.Slot = dev->get_slot(); - POOLMEM *changer_cmd = get_pool_memory(PM_FNAME); POOL_MEM results(PM_MESSAGE); - lock_changer(dcr); - if (!vol_name || !vol_name[0]) { - vol_name = "unknown"; + if (old_vol_name[0] == 0) { + if (dev->LoadedVolName[0]) { + old_vol_name = dev->LoadedVolName; + } else { + old_vol_name = "*Unknown*"; + } } + lock_changer(dcr); Jmsg(jcr, M_INFO, 0, - _("3307 Issuing autochanger \"unload slot %d, drive %d\" command for vol %s.\n"), - dev->get_slot(), dev->drive_index, vol_name); - - Dmsg3(dbglvl, "Issuing autochanger \"unload slot %d, drive %d\" command for vol %.\n", - dev->get_slot(), dev->drive_index, vol_name); + _("3307 Issuing autochanger \"unload Volume %s, Slot %d, Drive %d\" command.\n"), + old_vol_name, dev->get_slot(), dev->drive_index); + Dmsg3(0/*dbglvl*/, "Issuing autochanger \"unload Volume %s, Slot %d, Drive %d\" command.\n", + old_vol_name, dev->get_slot(), dev->drive_index); changer_cmd = edit_device_codes(dcr, changer_cmd, dcr->device->changer_command, "unload"); - dev->close(); + dev->close(dcr); Dmsg2(dbglvl, "close dev=%s reserve=%d\n", dev->print_name(), dev->num_reserved()); Dmsg1(dbglvl, "Run program=%s\n", changer_cmd); @@ -574,17 +597,19 @@ bool unload_dev(DCR *dcr, DEVICE *dev) if (stat != 0) { berrno be; be.set_errno(stat); - Jmsg(jcr, M_INFO, 0, _("3997 Bad autochanger \"unload slot %d, drive %d\" for vol %s: ERR=%s.\n"), - dev->get_slot(), dev->drive_index, vol_name, be.bstrerror()); - Dmsg5(dbglvl, "Error: load slot %d, drive %d, vol %s bad stats=%s.\nResults=%s\n", - dev->get_slot(), dev->drive_index, vol_name, + Jmsg(jcr, M_INFO, 0, _("3997 Bad autochanger \"unload Volume %s, Slot %d, Drive %d\": ERR=%s.\n"), + old_vol_name, dev->get_slot(), dev->drive_index, be.bstrerror()); + Dmsg5(dbglvl, "Error: unload Volume %s, Slot %d, Drive %d bad stats=%s.\nResults=%s\n", + old_vol_name, dev->get_slot(), dev->drive_index, be.bstrerror(), results.c_str()); ok = false; dev->clear_slot(); /* unknown */ } else { - Dmsg3(dbglvl, "Slot %d vol %s unloaded %s\n", dev->get_slot(), vol_name, dev->print_name()); + Dmsg3(dbglvl, "Volume %s, Slot %d unloaded %s\n", + old_vol_name, dev->get_slot(), dev->print_name()); dev->set_slot(0); /* unload OK, mark nothing loaded */ dev->clear_unload(); + dev->LoadedVolName[0] = 0; } unlock_changer(dcr); diff --git a/bacula/src/stored/bacula-sd.conf.in b/bacula/src/stored/bacula-sd.conf.in index f9948109ec..1c1d4a401c 100644 --- a/bacula/src/stored/bacula-sd.conf.in +++ b/bacula/src/stored/bacula-sd.conf.in @@ -10,7 +10,7 @@ # that dird.conf has corresponding changes. # # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @@ -19,6 +19,7 @@ Storage { # definition of myself SDPort = @sd_port@ # Director's port WorkingDirectory = "@working_dir@" Pid Directory = "@piddir@" + Plugin Directory = "@plugindir@" Maximum Concurrent Jobs = 20 } @@ -44,7 +45,7 @@ Director { # Note, for a list of additional Device templates please # see the directory /examples/devices # Or follow the following link: -# http://bacula.svn.sourceforge.net/viewvc/bacula/trunk/bacula/examples/devices/ +# http://www.bacula.org/git/cgit.cgi/bacula/tree/bacula/examples/devices?h=Branch-7.4 # # @@ -145,6 +146,15 @@ Device { # RandomAccess = no; # AutoChanger = yes # # +# # New alert command in Bacula 9.0.0 +# # Note: you must have the sg3_utils (rpms) or the +# # sg3-utils (deb) installed on your system. +# # and you must set the correct control device that +# # corresponds to the Archive Device +# Control Device = /dev/sg?? # must be SCSI ctl for /dev/nst0 +# Alert Command = "@scriptdir@/tapealert %l" +# +# # # # Enable the Alert command only if you have the mtx package loaded # # Note, apparently on some systems, tapeinfo resets the SCSI controller # # thus if you turn this on, make sure it does not reset your SCSI @@ -205,9 +215,18 @@ Device { # RemovableMedia = yes; # RandomAccess = no; # Maximum File Size = 4GB -## Changer Command = "@scriptdir@/mtx-changer %c %o %S %a %d" -## Changer Device = /dev/sg0 -## AutoChanger = yes +# Changer Command = "@scriptdir@/mtx-changer %c %o %S %a %d" +# Changer Device = /dev/sg0 +# AutoChanger = yes +# # +# # New alert command in Bacula 9.0.0 +# # Note: you must have the sg3_utils (rpms) or the +# # sg3-utils (deb) installed on your system. +# # and you must set the correct control device that +# # corresponds to the Archive Device +# Control Device = /dev/sg?? # must be SCSI ctl for @TAPEDRIVE@ +# Alert Command = "@scriptdir@/tapealert %l" +# # # Enable the Alert command only if you have the mtx package loaded ## Alert Command = "sh -c 'tapeinfo -f %c |grep TapeAlert|cat'" ## If you have smartctl, enable this, it has more info than tapeinfo @@ -226,9 +245,18 @@ Device { # RemovableMedia = yes; # RandomAccess = no; # Maximum File Size = 5GB -## Changer Command = "@scriptdir@/mtx-changer %c %o %S %a %d" -## Changer Device = /dev/sg0 -## AutoChanger = yes +# Changer Command = "@scriptdir@/mtx-changer %c %o %S %a %d" +# Changer Device = /dev/sg0 +# AutoChanger = yes +# # +# # New alert command in Bacula 9.0.0 +# # Note: you must have the sg3_utils (rpms) or the +# # sg3-utils (deb) installed on your system. +# # and you must set the correct control device that +# # corresponds to the Archive Device +# Control Device = /dev/sg?? # must be SCSI ctl for @TAPEDRIVE@ +# Alert Command = "@scriptdir@/tapealert %l" +# # # Enable the Alert command only if you have the mtx package loaded ## Alert Command = "sh -c 'tapeinfo -f %c |grep TapeAlert|cat'" ## If you have smartctl, enable this, it has more info than tapeinfo @@ -252,6 +280,15 @@ Device { # Hardware End of Medium = no # Fast Forward Space File = no # # +# # New alert command in Bacula 9.0.0 +# # Note: you must have the sg3_utils (rpms) or the +# # sg3-utils (deb) installed on your system. +# # and you must set the correct control device that +# # corresponds to the Archive Device +# Control Device = /dev/sg?? # must be SCSI ctl for /dev/rmt/1mnb +# Alert Command = "@scriptdir@/tapealert %l" +# +# # # # Enable the Alert command only if you have the mtx package loaded # Alert Command = "sh -c 'tapeinfo -f %c |grep TapeAlert|cat'" # If you have smartctl, enable this, it has more info than tapeinfo @@ -274,6 +311,15 @@ Device { # Backward Space Record = no # Fast Forward Space File = no # TWO EOF = yes +# # +# # New alert command in Bacula 9.0.0 +# # Note: you must have the sg3_utils (rpms) or the +# # sg3-utils (deb) installed on your system. +# # and you must set the correct control device that +# # corresponds to the Archive Device +# Control Device = /dev/sg?? # must be SCSI ctl for /dev/nsa1 +# Alert Command = "@scriptdir@/tapealert %l" +# # If you have smartctl, enable this, it has more info than tapeinfo # Alert Command = "sh -c 'smartctl -H -l error %c'" #} diff --git a/bacula/src/stored/bcopy.c b/bacula/src/stored/bcopy.c index 26bcde1a24..a268406652 100644 --- a/bacula/src/stored/bcopy.c +++ b/bacula/src/stored/bcopy.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +21,6 @@ * Program to copy a Bacula from one volume to another. * * Kern E. Sibbald, October 2002 - * */ #include "bacula.h" @@ -52,10 +51,6 @@ static CONFIG *config; void *start_heap; char *configfile = NULL; -STORES *me = NULL; /* our Global resource */ -bool forge_on = false; /* proceed inspite of I/O errors */ -pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; static void usage() { @@ -83,7 +78,9 @@ int main (int argc, char *argv[]) char *oVolumeName = NULL; bool ignore_label_errors = false; bool ok; + BtoolsAskDirHandler askdir_handler; + init_askdir_handler(&askdir_handler); setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); @@ -160,7 +157,7 @@ int main (int argc, char *argv[]) configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); + config = New(CONFIG()); parse_sd_config(config, configfile, M_ERROR_TERM); setup_me(); load_sd_plugins(me->plugin_directory); @@ -190,7 +187,7 @@ int main (int argc, char *argv[]) Dmsg0(100, "About to acquire device for writing\n"); /* For we must now acquire the device for writing */ out_dev->rLock(false); - if (!out_dev->open(out_jcr->dcr, OPEN_READ_WRITE)) { + if (!out_dev->open_device(out_jcr->dcr, OPEN_READ_WRITE)) { Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), out_dev->errmsg); out_dev->Unlock(); exit(1); @@ -215,8 +212,9 @@ int main (int argc, char *argv[]) free_jcr(in_jcr); free_jcr(out_jcr); - in_dev->term(); - out_dev->term(); + in_dev->term(NULL); + out_dev->term(NULL); + return 0; } @@ -348,32 +346,3 @@ static void get_session_record(DEVICE *dev, DEV_RECORD *rec, SESSION_LABEL *sess rtype, rec->VolSessionId, rec->VolSessionTime, rec->Stream, rec->data_len); } } - - -/* Dummies to replace askdir.c */ -bool dir_find_next_appendable_volume(DCR *dcr) { return 1;} -bool dir_update_volume_info(DCR *dcr, bool relabel, bool update_LastWritten) { return 1; } -bool dir_create_jobmedia_record(DCR *dcr, bool zero) { return 1; } -bool flush_jobmedia_queue(JCR *jcr) { return true; } -bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { return 1; } -bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { return 1;} -bool dir_send_job_status(JCR *jcr) {return 1;} - - -bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /*writing*/) -{ - DEVICE *dev = dcr->dev; - fprintf(stderr, _("Mount Volume \"%s\" on device %s and press return when ready: "), - dcr->VolumeName, dev->print_name()); - dev->close(); - getchar(); - return true; -} - -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) -{ - Dmsg0(100, "Fake dir_get_volume_info\n"); - dcr->setVolCatName(dcr->VolumeName); - Dmsg2(500, "Vol=%s VolType=%d\n", dcr->getVolCatName(), dcr->VolCatInfo.VolCatType); - return 1; -} diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c index a300792f3c..6dd1b23dcd 100644 --- a/bacula/src/stored/bextract.c +++ b/bacula/src/stored/bextract.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +21,6 @@ * Dumb program to extract files from a Bacula backup. * * Kern E. Sibbald, MM - * */ #include "bacula.h" @@ -69,11 +68,7 @@ static CONFIG *config; void *start_heap; char *configfile = NULL; -STORES *me = NULL; /* our Global resource */ -bool forge_on = false; bool skip_extract = false; -pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; static void usage() { @@ -103,7 +98,9 @@ int main (int argc, char *argv[]) FILE *fd; char line[1000]; bool got_inc = false; + BtoolsAskDirHandler askdir_handler; + init_askdir_handler(&askdir_handler); setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); @@ -127,7 +124,6 @@ int main (int argc, char *argv[]) case 'b': /* bootstrap file */ bsr = parse_bsr(NULL, optarg); -// dump_bsr(bsr, true); break; case 'T': /* Send debug to trace file */ @@ -145,10 +141,18 @@ int main (int argc, char *argv[]) if (*optarg == 't') { dbg_timestamp = true; } else { + char *p; + /* We probably find a tag list -d 10,sql,bvfs */ + if ((p = strchr(optarg, ',')) != NULL) { + *p = 0; + } debug_level = atoi(optarg); if (debug_level <= 0) { debug_level = 1; } + if (p) { + debug_parse_tags(p+1, &debug_level_tags); + } } break; @@ -161,9 +165,6 @@ int main (int argc, char *argv[]) } while (fgets(line, sizeof(line), fd) != NULL) { strip_trailing_junk(line); - if (line[0] == 0) { /* skip blank lines */ - continue; - } Dmsg1(900, "add_exclude %s\n", line); add_fname_to_exclude_list(ff, line); } @@ -179,9 +180,6 @@ int main (int argc, char *argv[]) } while (fgets(line, sizeof(line), fd) != NULL) { strip_trailing_junk(line); - if (line[0] == 0) { /* skip blank lines */ - continue; - } Dmsg1(900, "add_include %s\n", line); add_fname_to_include_list(ff, 0, line); } @@ -219,7 +217,7 @@ int main (int argc, char *argv[]) configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); + config = New(CONFIG()); parse_sd_config(config, configfile, M_ERROR_TERM); setup_me(); load_sd_plugins(me->plugin_directory); @@ -254,7 +252,7 @@ static void do_extract(char *devname) enable_backup_privileges(NULL, 1); - jcr = setup_jcr("bextract", devname, bsr, VolumeName, SD_READ, false/*read dedup data*/); + jcr = setup_jcr("bextract", devname, bsr, VolumeName, SD_READ, true/*read dedup data*/); if (!jcr) { exit(1); } @@ -292,7 +290,7 @@ static void do_extract(char *devname) release_device(dcr); free_attr(attr); free_jcr(jcr); - dev->term(); + dev->term(NULL); free_pool_memory(curr_fname); printf(_("%u files restored.\n"), num_files); @@ -442,7 +440,6 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) case STREAM_FILE_DATA: case STREAM_SPARSE_DATA: case STREAM_WIN32_DATA: - if (extract) { if (rec->maskedStream == STREAM_SPARSE_DATA) { ser_declare; @@ -451,12 +448,13 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) wsize = rec->data_len - OFFSET_FADDR_SIZE; ser_begin(rec->data, OFFSET_FADDR_SIZE); unser_uint64(faddr); - if (fileAddr != faddr) { + /* We seek only for real SPARSE data, not for OFFSET option */ + if ((rec->Stream & STREAM_BIT_OFFSETS) == 0 && fileAddr != faddr) { fileAddr = faddr; if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { berrno be; - Emsg2(M_ERROR_TERM, 0, _("Seek error on %s: %s\n"), - attr->ofname, be.bstrerror()); + Emsg3(M_ERROR_TERM, 0, _("Seek error Addr=%llu on %s: %s\n"), + fileAddr, attr->ofname, be.bstrerror()); } } } else { @@ -479,7 +477,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) uLong compress_len = compress_buf_size; int stat = Z_BUF_ERROR; - if (rec->maskedStream == STREAM_SPARSE_GZIP_DATA) { + if (rec->maskedStream == STREAM_SPARSE_DATA) { ser_declare; uint64_t faddr; char ec1[50]; @@ -487,7 +485,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) wsize = rec->data_len - OFFSET_FADDR_SIZE; ser_begin(rec->data, OFFSET_FADDR_SIZE); unser_uint64(faddr); - if (fileAddr != faddr) { + if ((rec->Stream & STREAM_BIT_OFFSETS) == 0 && fileAddr != faddr) { fileAddr = faddr; if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { berrno be; @@ -544,7 +542,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) int r, real_compress_len; #endif - if (rec->maskedStream == STREAM_SPARSE_COMPRESSED_DATA) { + if (rec->maskedStream == STREAM_SPARSE_DATA) { ser_declare; uint64_t faddr; char ec1[50]; @@ -552,7 +550,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) wsize = rec->data_len - OFFSET_FADDR_SIZE; ser_begin(rec->data, OFFSET_FADDR_SIZE); unser_uint64(faddr); - if (fileAddr != faddr) { + if ((rec->Stream & STREAM_BIT_OFFSETS) == 0 && fileAddr != faddr) { fileAddr = faddr; if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { berrno be; @@ -670,31 +668,3 @@ bail_out: } return ret; } - -/* Dummies to replace askdir.c */ -bool dir_find_next_appendable_volume(DCR *dcr) { return 1;} -bool dir_update_volume_info(DCR *dcr, bool relabel, bool update_LastWritten) { return 1; } -bool dir_create_jobmedia_record(DCR *dcr, bool zero) { return 1; } -bool flush_jobmedia_queue(JCR *jcr) { return true; } -bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { return 1; } -bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { return 1;} -bool dir_send_job_status(JCR *jcr) {return 1;} - - -bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /*writing*/) -{ - DEVICE *dev = dcr->dev; - fprintf(stderr, _("Mount Volume \"%s\" on device %s and press return when ready: "), - dcr->VolumeName, dev->print_name()); - dev->close(); - getchar(); - return true; -} - -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) -{ - Dmsg0(100, "Fake dir_get_volume_info\n"); - dcr->setVolCatName(dcr->VolumeName); - Dmsg2(500, "Vol=%s VolType=%d\n", dcr->getVolCatName(), dcr->VolCatInfo.VolCatType); - return 1; -} diff --git a/bacula/src/stored/block.c b/bacula/src/stored/block.c index 3bc9dae9e6..06c1320a8d 100644 --- a/bacula/src/stored/block.c +++ b/bacula/src/stored/block.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -22,7 +22,6 @@ * * Kern Sibbald, March MMI * added BB02 format October MMII - * */ @@ -35,6 +34,8 @@ static const bool debug_block_checksum = true; static const bool debug_block_checksum = false; #endif +static int debug_io_error = 0; /* # blocks to write before creating I/O error */ + #ifdef NO_TAPE_WRITE_TEST static const bool no_tape_write_test = true; #else @@ -42,12 +43,9 @@ static const bool no_tape_write_test = false; #endif -bool do_new_file_bookkeeping(DCR *dcr); -//bool do_dvd_size_checks(DCR *dcr); -void reread_last_block(DCR *dcr); uint32_t get_len_and_clear_block(DEV_BLOCK *block, DEVICE *dev, uint32_t &pad); uint32_t ser_block_header(DEV_BLOCK *block, bool do_checksum); -bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block); +bool unser_block_header(DCR *dcr, DEVICE *dev, DEV_BLOCK *block); /* * Write a block to the device, with locking and unlocking @@ -62,7 +60,7 @@ bool DCR::write_block_to_device(bool final) DCR *dcr = this; if (dcr->spooling) { - Dmsg0(200, "Write to spool\n"); + Dmsg0(250, "Write to spool\n"); ok = write_block_to_spool_file(dcr); return ok; } @@ -79,13 +77,28 @@ bool DCR::write_block_to_device(bool final) Dmsg1(500, "Write block to dev=%p\n", dcr->dev); if (!write_block_to_dev()) { - Dmsg1(40, "*** Failed write_block_to_dev block=%p\n", block); + Dmsg2(40, "*** Failed write_block_to_dev adata=%d block=%p\n", + block->adata, block); if (job_canceled(jcr) || jcr->getJobType() == JT_SYSTEM) { ok = false; Dmsg2(40, "cancel=%d or SYSTEM=%d\n", job_canceled(jcr), jcr->getJobType() == JT_SYSTEM); } else { - ok = fixup_device_block_write_error(dcr); + bool was_adata = false; + if (was_adata) { + dcr->set_ameta(); + was_adata = true; + } + /* Flush any existing JobMedia info */ + if (!(ok = dir_create_jobmedia_record(dcr))) { + Jmsg(jcr, M_FATAL, 0, _("Error writing JobMedia record to catalog.\n")); + } else { + Dmsg1(40, "Calling fixup_device was_adata=%d...\n", was_adata); + ok = fixup_device_block_write_error(dcr); + } + if (was_adata) { + dcr->set_adata(); + } } } if (ok && final && !dir_create_jobmedia_record(dcr)) { @@ -114,6 +127,8 @@ bool DCR::write_block_to_dev() DCR *dcr = this; uint32_t checksum; uint32_t pad; /* padding or zeros written */ + boffset_t pos; + char ed1[50]; if (no_tape_write_test) { empty_block(block); @@ -122,19 +137,23 @@ bool DCR::write_block_to_dev() if (job_canceled(jcr)) { return false; } + if (!dev->enabled) { + Jmsg1(jcr, M_FATAL, 0, _("Cannot write block. Device is disabled. dev=%s\n"), dev->print_name()); + return false; + } - Dmsg3(200, "fd=%d bufp-buf=%d binbuf=%d\n", dev->fd(), + ASSERT2(block->adata == dev->adata, "Block and dev adata not same"); + Dmsg4(200, "fd=%d adata=%d bufp-buf=%d binbuf=%d\n", dev->fd(), block->adata, block->bufp-block->buf, block->binbuf); ASSERT2(block->binbuf == ((uint32_t)(block->bufp - block->buf)), "binbuf badly set"); if (is_block_empty(block)) { /* Does block have data in it? */ - Dmsg0(150, "return write_block_to_dev no data to write\n"); + Dmsg1(50, "return write_block_to_dev no adata=%d data to write\n", block->adata); return true; } - dump_block(block, "before write"); if (dev->at_weot()) { - Dmsg0(50, "==== FATAL: At EOM with ST_WEOT.\n"); + Dmsg1(50, "==== FATAL: At EOM with ST_WEOT. adata=%d.\n", dev->adata); dev->dev_errno = ENOSPC; Jmsg1(jcr, M_FATAL, 0, _("Cannot write block. Device at EOM. dev=%s\n"), dev->print_name()); return false; @@ -154,50 +173,19 @@ bool DCR::write_block_to_dev() wlen = get_len_and_clear_block(block, dev, pad); block->block_len = wlen; + dev->updateVolCatPadding(pad); checksum = ser_block_header(block, dev->do_checksum()); - if (user_volume_size_reached(dcr, true)) { - Dmsg0(40, "Calling terminate_writing_volume\n"); - terminate_writing_volume(dcr); - reread_last_block(dcr); /* Only used on tapes */ - dev->dev_errno = ENOSPC; + if (!dev->do_size_checks(dcr, block)) { + Dmsg0(50, "Size check triggered. Cannot write block.\n"); return false; } - /* - * Limit maximum File size on volume to user specified value. - * In practical terms, this means to put an EOF mark on - * a tape after every X bytes. This effectively determines - * how many index records we have (JobMedia). If you set - * max_file_size too small, it will cause a lot of shoe-shine - * on very fast modern tape (LTO-3 and above). - */ - if ((dev->max_file_size > 0) && - (dev->file_size+block->binbuf) >= dev->max_file_size) { - dev->file_size = 0; /* reset file size */ - - if (!dev->weof(1)) { /* write eof */ - Dmsg0(50, "WEOF error in max file size.\n"); - Jmsg(jcr, M_FATAL, 0, _("Unable to write EOF. ERR=%s\n"), - dev->bstrerror()); - Dmsg0(40, "Calling terminate_writing_volume\n"); - terminate_writing_volume(dcr); - dev->dev_errno = ENOSPC; - return false; - } - if (!write_ansi_ibm_labels(dcr, ANSI_EOF_LABEL, dev->VolHdr.VolumeName)) { - return false; - } - - if (!do_new_file_bookkeeping(dcr)) { - /* Error message already sent */ - return false; - } - } - dev->updateVolCatWrites(1); + dump_block(dev, block, "before write"); + #ifdef DEBUG_BLOCK_ZEROING uint32_t *bp = (uint32_t *)block->buf; if (bp[0] == 0 && bp[1] == 0 && bp[2] == 0 && block->buf[12] == 0) { @@ -205,6 +193,30 @@ bool DCR::write_block_to_dev() } #endif + /* + * If Adata block, we must seek to the correct address + */ + if (block->adata) { + ASSERT(dcr->dev->adata); + uint64_t cur = dev->lseek(dcr, 0, SEEK_CUR); + /* If we are going to create a hole, record it */ + if (block->BlockAddr != cur) { + dev->lseek(dcr, block->BlockAddr, SEEK_SET); + Dmsg4(100, "Adata seek BlockAddr from %lld to %lld = %lld bytes adata_addr=%lld\n", + cur, block->BlockAddr, block->BlockAddr - cur, dev->adata_addr); + /* Insanity check */ + if (block->BlockAddr > cur) { + dev->updateVolCatHoleBytes(block->BlockAddr - cur); + } else if (block->BlockAddr < cur) { + Pmsg5(000, "Vol=%s cur=%lld BlockAddr=%lld adata=%d block=%p\n", + dev->getVolCatName(), cur, block->BlockAddr, block->adata, block); + Jmsg3(jcr, M_FATAL, 0, "Bad seek on adata Vol=%s BlockAddr=%lld DiskAddr=%lld. Multiple simultaneous Jobs?\n", + dev->getVolCatName(), block->BlockAddr, cur); + //Pmsg2(000, "HoleBytes would go negative cur=%lld blkaddr=%lld\n", cur, block->BlockAddr); + } + } + } + /* * Do write here, make a somewhat feeble attempt to recover from * I/O errors, or from the OS telling us it is busy. @@ -212,6 +224,8 @@ bool DCR::write_block_to_dev() int retry = 0; errno = 0; stat = 0; + /* ***FIXME**** remove next line debug */ + pos = dev->lseek(dcr, 0, SEEK_CUR); do { if (retry > 0 && stat == -1 && errno == EBUSY) { berrno be; @@ -221,17 +235,21 @@ bool DCR::write_block_to_dev() dev->clrerror(-1); } stat = dev->write(block->buf, (size_t)wlen); - Dmsg4(110, "Write() BlockAddr=%lld NextAddr=%lld Vol=%s wlen=%d\n", - block->BlockAddr, dev->lseek(dcr, 0, SEEK_CUR), - dev->VolHdr.VolumeName, wlen); + Dmsg4(100, "%s write() BlockAddr=%lld wlen=%d Vol=%s wlen=%d\n", + block->adata?"Adata":"Ameta", block->BlockAddr, wlen, + dev->VolHdr.VolumeName); } while (stat == -1 && (errno == EBUSY || errno == EIO) && retry++ < 3); + /* ***FIXME*** remove 2 lines debug */ + Dmsg2(100, "Wrote %d bytes at %s\n", wlen, dev->print_addr(ed1, sizeof(ed1), pos)); + dump_block(dev, block, "After write"); + if (debug_block_checksum) { uint32_t achecksum = ser_block_header(block, dev->do_checksum()); if (checksum != achecksum) { Jmsg2(jcr, M_ERROR, 0, _("Block checksum changed during write: before=%u after=%u\n"), checksum, achecksum); - dump_block(block, "with checksum error"); + dump_block(dev, block, "with checksum error"); } } @@ -241,6 +259,16 @@ bool DCR::write_block_to_dev() } #endif + if (debug_io_error) { + debug_io_error--; + if (debug_io_error == 1) { /* trigger error */ + stat = -1; + dev->dev_errno = EIO; + errno = EIO; + debug_io_error = 0; /* turn off trigger */ + } + } + if (stat != (ssize_t)wlen) { /* Some devices simply report EIO when the volume is full. * With a little more thought we may be able to check @@ -255,9 +283,17 @@ bool DCR::write_block_to_dev() dev->dev_errno = ENOSPC; /* out of space */ } if (dev->dev_errno != ENOSPC) { + int etype = M_ERROR; + if (block->adata) { + etype = M_FATAL; + } dev->VolCatInfo.VolCatErrors++; - Jmsg4(jcr, M_ERROR, 0, _("Write error at %u:%u on device %s. ERR=%s.\n"), - dev->file, dev->block_num, dev->print_name(), be.bstrerror()); + Jmsg4(jcr, etype, 0, _("Write error at %s on device %s Vol=%s. ERR=%s.\n"), + dev->print_addr(ed1, sizeof(ed1)), dev->print_name(), + dev->getVolCatName(), be.bstrerror()); + if (dev->get_tape_alerts(this)) { + dev->show_tape_alerts(this, list_long, list_last, alert_callback); + } } } else { dev->dev_errno = ENOSPC; /* out of space */ @@ -266,14 +302,14 @@ bool DCR::write_block_to_dev() dev->update_freespace(); if (dev->is_freespace_ok() && dev->free_space < dev->min_free_space) { dev->set_nospace(); - Jmsg(jcr, M_WARNING, 0, _("Out of freespace caused End of Volume \"%s\" at %u:%u on device %s. Write of %u bytes got %d.\n"), + Jmsg(jcr, M_WARNING, 0, _("Out of freespace caused End of Volume \"%s\" at %s on device %s. Write of %u bytes got %d.\n"), dev->getVolCatName(), - dev->file, dev->block_num, dev->print_name(), wlen, stat); + dev->print_addr(ed1, sizeof(ed1)), dev->print_name(), wlen, stat); } else { dev->clear_nospace(); - Jmsg(jcr, M_INFO, 0, _("End of Volume \"%s\" at %u:%u on device %s. Write of %u bytes got %d.\n"), + Jmsg(jcr, M_INFO, 0, _("End of Volume \"%s\" at %s on device %s. Write of %u bytes got %d.\n"), dev->getVolCatName(), - dev->file, dev->block_num, dev->print_name(), wlen, stat); + dev->print_addr(ed1, sizeof(ed1)), dev->print_name(), wlen, stat); } } if (chk_dbglvl(100)) { @@ -294,50 +330,68 @@ bool DCR::write_block_to_dev() /* We successfully wrote the block, now do housekeeping */ Dmsg2(1300, "VolCatBytes=%lld newVolCatBytes=%lld\n", dev->VolCatInfo.VolCatBytes, (dev->VolCatInfo.VolCatBytes+wlen)); - dev->updateVolCatBytes(wlen); + if (!dev->setVolCatAdataBytes(block->BlockAddr + wlen)) { + dev->updateVolCatBytes(wlen); + Dmsg3(200, "AmetaBytes=%lld AdataBytes=%lld Bytes=%lld\n", + dev->VolCatInfo.VolCatAmetaBytes, dev->VolCatInfo.VolCatAdataBytes, dev->VolCatInfo.VolCatBytes); + } dev->updateVolCatBlocks(1); - dev->EndBlock = dev->block_num; - dev->EndFile = dev->file; dev->LastBlock = block->BlockNumber; block->BlockNumber++; /* Update dcr values */ if (dev->is_tape()) { - if (dcr->EndFile != dev->EndFile || dev->EndBlock > dcr->EndBlock) { - dcr->EndBlock = dev->EndBlock; - dcr->EndFile = dev->EndFile; + dev->EndAddr = dev->get_full_addr(); + if (dcr->EndAddr < dev->EndAddr) { + dcr->EndAddr = dev->EndAddr; } dev->block_num++; } else { - /* Save address of block just written */ + /* Save address of byte just written */ uint64_t addr = dev->file_addr + wlen - 1; - if (dcr->EndFile == (uint32_t)(addr >> 32) && - (uint32_t)addr < dcr->EndBlock) { - Pmsg4(000, "Possible incorrect EndBlock oldEndBlock=%u newEndBlock=%u oldEndFile=%u newEndFile=%u\n", - dcr->EndBlock, (uint32_t)addr, dcr->EndFile, (uint32_t)(addr >> 32)); + if (dev->is_indexed()) { + uint64_t full_addr = dev->get_full_addr(addr); + if (full_addr < dcr->EndAddr) { + Pmsg2(000, "Possible incorrect EndAddr oldEndAddr=%llu newEndAddr=%llu\n", + dcr->EndAddr, full_addr); + } + dcr->EndAddr = full_addr; + } + + if (dev->adata) { + /* We really should use file_addr, but I am not sure it is correctly set */ + Dmsg3(100, "Set BlockAddr from %lld to %lld adata_addr=%lld\n", + block->BlockAddr, block->BlockAddr + wlen, dev->adata_addr); + block->BlockAddr += wlen; + dev->adata_addr = block->BlockAddr; } else { - dcr->EndBlock = (uint32_t)addr; - dcr->EndFile = (uint32_t)(addr >> 32); + block->BlockAddr = dev->get_full_addr() + wlen; + } + } + if (dev->is_indexed()) { + if (dcr->VolMediaId != dev->VolCatInfo.VolMediaId) { + Dmsg7(100, "JobMedia Vol=%s wrote=%d MediaId=%lld FI=%lu LI=%lu StartAddr=%lld EndAddr=%lld Wrote\n", + dcr->VolumeName, dcr->WroteVol, dcr->VolMediaId, + dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartAddr, dcr->EndAddr); + } + dcr->VolMediaId = dev->VolCatInfo.VolMediaId; + Dmsg3(150, "VolFirstIndex=%d blockFirstIndex=%d Vol=%s\n", + dcr->VolFirstIndex, block->FirstIndex, dcr->VolumeName); + if (dcr->VolFirstIndex == 0 && block->FirstIndex > 0) { + dcr->VolFirstIndex = block->FirstIndex; + } + if (block->LastIndex > (int32_t)dcr->VolLastIndex) { + dcr->VolLastIndex = block->LastIndex; } - dev->block_num = (uint32_t)addr; - dev->file = (uint32_t)(addr >> 32); - block->BlockAddr = dev->file_addr + wlen; - Dmsg3(150, "Set block->BlockAddr=%lld wlen=%d block=%x\n", - block->BlockAddr, wlen, block); - } - dcr->VolMediaId = dev->VolCatInfo.VolMediaId; - Dmsg3(150, "VolFirstIndex=%d blockFirstIndex=%d Vol=%s\n", - dcr->VolFirstIndex, block->FirstIndex, dcr->VolumeName); - if (dcr->VolFirstIndex == 0 && block->FirstIndex > 0) { - dcr->VolFirstIndex = block->FirstIndex; - } - if (block->LastIndex > (int32_t)dcr->VolLastIndex) { - dcr->VolLastIndex = block->LastIndex; - } - dcr->WroteVol = true; + dcr->WroteVol = true; + } + dev->file_addr += wlen; /* update file address */ dev->file_size += wlen; - dev->part_size += wlen; + dev->usage += wlen; /* update usage counter */ + if (dev->part > 0) { + dev->part_size += wlen; + } dev->setVolCatInfo(false); /* Needs update */ Dmsg2(1300, "write_block: wrote block %d bytes=%d\n", dev->block_num, wlen); @@ -358,18 +412,22 @@ bool DCR::read_block_from_device(bool check_block_numbers) dev->rLock(false); ok = read_block_from_dev(check_block_numbers); dev->rUnlock(); - Dmsg0(250, "Leave read_block_from_device\n"); + Dmsg1(250, "Leave read_block_from_device. ok=%d\n", ok); return ok; } static void set_block_position(DCR *dcr, DEVICE *dev, DEV_BLOCK *block) { + /* Used also by the Single Item Restore code to locate a particular block */ if (dev->is_tape()) { - block->Block = dev->block_num; - block->File = dev->file; - } else { - block->Block = (uint32_t)dev->file_addr; - block->File = (uint32_t)(dev->file_addr >> 32); + block->BlockAddr = dev->get_full_addr(); + + } else if (!dev->adata) { /* TODO: See if we just use !dev->adata for tapes */ + /* Note: we only update the DCR values for ameta blocks + * because all the indexing (JobMedia) is done with + * ameta blocks/records, which may point to adata. + */ + block->BlockAddr = dev->get_full_addr(); } block->RecNum = 0; } @@ -378,6 +436,8 @@ static void set_block_position(DCR *dcr, DEVICE *dev, DEV_BLOCK *block) * Read the next block into the block structure and unserialize * the block header. For a file, the block may be partially * or completely in the current buffer. + * Note: in order for bscan to generate correct JobMedia records + * we must be careful to update the EndAddr of the last byte read. */ bool DCR::read_block_from_dev(bool check_block_numbers) { @@ -385,25 +445,35 @@ bool DCR::read_block_from_dev(bool check_block_numbers) int looping; int retry; DCR *dcr = this; + boffset_t pos; + char ed1[50]; if (job_canceled(jcr)) { Mmsg(dev->errmsg, _("Job failed or canceled.\n")); + Dmsg1(000, "%s", dev->errmsg); block->read_len = 0; return false; } + if (!dev->enabled) { + Mmsg(dev->errmsg, _("Cannot write block. Device is disabled. dev=%s\n"), dev->print_name()); + Jmsg1(jcr, M_FATAL, 0, "%s", dev->errmsg); + return false; + } if (dev->at_eot()) { - Mmsg(dev->errmsg, _("Attempt to read past end of tape or file.\n")); + Mmsg(dev->errmsg, _("At EOT: attempt to read past end of Volume.\n")); + Dmsg1(000, "%s", dev->errmsg); block->read_len = 0; return false; } looping = 0; - Dmsg1(250, "Full read in read_block_from_device() len=%d\n", block->buf_len); if (!dev->is_open()) { Mmsg4(dev->errmsg, _("Attempt to read closed device: fd=%d at file:blk %u:%u on device %s\n"), dev->fd(), dev->file, dev->block_num, dev->print_name()); Jmsg(dcr->jcr, M_FATAL, 0, "%s", dev->errmsg); + Pmsg4(000, "Fatal: dev=%p dcr=%p adata=%d bytes=%lld\n", dev, dcr, dev->adata, + VolCatInfo.VolCatAdataRBytes); Pmsg1(000, "%s", dev->errmsg); block->read_len = 0; return false; @@ -416,33 +486,32 @@ reread: dev->dev_errno = EIO; Mmsg1(dev->errmsg, _("Block buffer size looping problem on device %s\n"), dev->print_name()); + Dmsg1(000, "%s", dev->errmsg); Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); block->read_len = 0; return false; } - dump_block(block, "before read"); - -#ifdef xxxBUILD_DVD - /* Check for DVD part file end */ - if (dev->at_eof() && dev->is_dvd() && dev->num_dvd_parts > 0 && - dev->part <= dev->num_dvd_parts) { - Dmsg0(400, "Call dvd_open_next_part\n"); - if (dvd_open_next_part(dcr) < 0) { - Mmsg3(dev->errmsg, _("Unable to open device part=%d %s: ERR=%s\n"), - dev->part, dev->print_name(), dev->bstrerror()); - Jmsg(dcr->jcr, M_FATAL, 0, "%s", dev->errmsg); - dev->dev_errno = EIO; - return false; + /* See if we must open another part */ + if (dev->at_eof() && !dev->open_next_part(dcr)) { + if (dev->at_eof()) { /* EOF just seen? */ + dev->set_eot(); /* yes, error => EOT */ } + return false; } -#endif /* xxxBUILD_DVD */ retry = 0; errno = 0; stat = 0; - boffset_t pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); /* get curr pos */ + if (dev->adata) { + dev->lseek(dcr, dcr->block->BlockAddr, SEEK_SET); + } + { /* debug block */ + pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); /* get curr pos */ + Dmsg2(200, "Pos for read=%s %lld\n", + dev->print_addr(ed1, sizeof(ed1), pos), pos); + } do { if ((retry > 0 && stat == -1 && errno == EBUSY)) { berrno be; @@ -454,77 +523,93 @@ reread: stat = dev->read(block->buf, (size_t)block->buf_len); } while (stat == -1 && (errno == EBUSY || errno == EINTR || errno == EIO) && retry++ < 3); - Dmsg3(110, "Read() vol=%s nbytes=%d addr=%lld\n", - dev->VolHdr.VolumeName, stat, pos); + Dmsg4(110, "Read() adata=%d vol=%s nbytes=%d pos=%lld\n", + block->adata, dev->VolHdr.VolumeName, stat, pos); if (stat < 0) { berrno be; dev->clrerror(-1); Dmsg2(90, "Read device fd=%d got: ERR=%s\n", dev->fd(), be.bstrerror()); block->read_len = 0; - if (dev->file == 0 && dev->block_num == 0) { /* Attempt to read label */ - Mmsg(dev->errmsg, _("The Volume=%s on device=%s appears to be unlabeled.\n"), - "", dev->VolCatInfo.VolCatName, dev->print_name()); + if (reading_label) { /* Trying to read Volume label */ + Mmsg(dev->errmsg, _("The %sVolume=%s on device=%s appears to be unlabeled.%s\n"), + dev->adata?"adata ":"", VolumeName, dev->print_name(), + dev->is_fs_nearly_full(1048576)?" Notice that the filesystem is nearly full.":""); } else { - Mmsg5(dev->errmsg, _("Read error on fd=%d at file:blk %u:%u on device %s. ERR=%s.\n"), - dev->fd(), dev->file, dev->block_num, dev->print_name(), be.bstrerror()); + Mmsg4(dev->errmsg, _("Read error on fd=%d at addr=%s on device %s. ERR=%s.\n"), + dev->fd(), dev->print_addr(ed1, sizeof(ed1)), dev->print_name(), be.bstrerror()); } Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); + if (dev->get_tape_alerts(this)) { + dev->show_tape_alerts(this, list_long, list_last, alert_callback); + } if (dev->at_eof()) { /* EOF just seen? */ dev->set_eot(); /* yes, error => EOT */ } return false; } if (stat == 0) { /* Got EOF ! */ - if (dev->file == 0 && dev->block_num == 0) { /* Attempt to read label */ - Mmsg3(dev->errmsg, _("The %sVolume=%s on device=%s appears to be unlabeled.\n"), - "", dev->VolCatInfo.VolCatName, dev->print_name()); + pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); /* get curr pos */ + pos = dev->get_full_addr(pos); + if (reading_label) { /* Trying to read Volume label */ + Mmsg4(dev->errmsg, _("The %sVolume=%s on device=%s appears to be unlabeled.%s\n"), + dev->adata?"adata ":"", VolumeName, dev->print_name(), + dev->is_fs_nearly_full(1048576)?" Notice tha the filesystem is nearly full.":""); } else { - Mmsg4(dev->errmsg, _("Read zero %sbytes Vol=%s at %lld on device %s.\n"), - "", dev->VolCatInfo.VolCatName, pos, dev->print_name()); + Mmsg4(dev->errmsg, _("Read zero %sbytes Vol=%s at %s on device %s.\n"), + dev->adata?"adata ":"", dev->VolCatInfo.VolCatName, + dev->print_addr(ed1, sizeof(ed1), pos), dev->print_name()); } - dev->block_num = 0; block->read_len = 0; Dmsg1(100, "%s", dev->errmsg); - if (dev->at_eof()) { /* EOF already read? */ - dev->set_eot(); /* yes, 2 EOFs => EOT */ - return false; + if (dev->at_eof()) { /* EOF just seen? */ + dev->set_eot(); /* yes, error => EOT */ } dev->set_ateof(); - Dmsg2(150, "==== Read zero bytes. vol=%s at %lld\n", dev->VolCatInfo.VolCatName, pos); + dev->file_addr = 0; + dev->EndAddr = pos; + if (dcr->EndAddr < dev->EndAddr) { + dcr->EndAddr = dev->EndAddr; + } + Dmsg3(150, "==== Read zero bytes. adata=%d vol=%s at %s\n", dev->adata, + dev->VolCatInfo.VolCatName, dev->print_addr(ed1, sizeof(ed1), pos)); return false; /* return eof */ } /* Continue here for successful read */ block->read_len = stat; /* save length read */ - if (block->read_len == 80 && - (dcr->VolCatInfo.LabelType != B_BACULA_LABEL || - dcr->device->label_type != B_BACULA_LABEL)) { - /* ***FIXME*** should check label */ - Dmsg2(100, "Ignore 80 byte ANSI label at %u:%u\n", dev->file, dev->block_num); - dev->clear_eof(); - goto reread; /* skip ANSI/IBM label */ - } + if (block->adata) { + block->binbuf = block->read_len; + block->block_len = block->read_len; + } else { + if (block->read_len == 80 && + (dcr->VolCatInfo.LabelType != B_BACULA_LABEL || + dcr->device->label_type != B_BACULA_LABEL)) { + /* ***FIXME*** should check label */ + Dmsg2(100, "Ignore 80 byte ANSI label at %u:%u\n", dev->file, dev->block_num); + dev->clear_eof(); + goto reread; /* skip ANSI/IBM label */ + } - if (block->read_len < BLKHDR2_LENGTH) { - dev->dev_errno = EIO; - Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Very short block of %d bytes on device %s discarded.\n"), - dev->file, dev->block_num, block->read_len, dev->print_name()); - Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); - dev->set_short_block(); - block->read_len = block->binbuf = 0; - Dmsg2(50, "set block=%p binbuf=%d\n", block, block->binbuf); - return false; /* return error */ - } + if (block->read_len < BLKHDR2_LENGTH) { + dev->dev_errno = EIO; + Mmsg3(dev->errmsg, _("Volume data error at %s! Very short block of %d bytes on device %s discarded.\n"), + dev->print_addr(ed1, sizeof(ed1)), block->read_len, dev->print_name()); + Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); + dev->set_short_block(); + block->read_len = block->binbuf = 0; + Dmsg2(50, "set block=%p binbuf=%d\n", block, block->binbuf); + return false; /* return error */ + } - //BlockNumber = block->BlockNumber + 1; - if (!unser_block_header(jcr, dev, block)) { - if (forge_on) { - dev->file_addr += block->read_len; - dev->file_size += block->read_len; - goto reread; + if (!unser_block_header(this, dev, block)) { + if (forge_on) { + dev->file_addr += block->read_len; + dev->file_size += block->read_len; + goto reread; + } + return false; } - return false; } /* @@ -532,7 +617,7 @@ reread: * re-reading the block, allocate a buffer of the correct size, * and go re-read. */ - Dmsg2(150, "block_len=%d buf_len=%d\n", block->block_len, block->buf_len); + Dmsg3(150, "adata=%d block_len=%d buf_len=%d\n", block->adata, block->block_len, block->buf_len); if (block->block_len > block->buf_len) { dev->dev_errno = EIO; Mmsg2(dev->errmsg, _("Block length %u is greater than buffer %u. Attempting recovery.\n"), @@ -544,7 +629,9 @@ reread: Dmsg0(250, "BSR for reread; block too big for buffer.\n"); if (!dev->bsr(1)) { Mmsg(dev->errmsg, "%s", dev->bstrerror()); - Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); + if (dev->errmsg[0]) { + Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); + } block->read_len = 0; return false; } @@ -583,32 +670,32 @@ reread: dev->updateVolCatReads(1); dev->updateVolCatReadBytes(block->read_len); - dev->EndBlock = dev->block_num; - dev->EndFile = dev->file; - dev->block_num++; - /* Update dcr values */ if (dev->is_tape()) { - if (dcr->EndFile != dev->EndFile || dev->EndBlock > dcr->EndBlock) { - dcr->EndBlock = dev->EndBlock; - dcr->EndFile = dev->EndFile; + dev->EndAddr = dev->get_full_addr(); + if (dcr->EndAddr < dev->EndAddr) { + dcr->EndAddr = dev->EndAddr; } + dev->block_num++; } else { /* We need to take care about a short block in EndBlock/File * computation */ uint32_t len = MIN(block->read_len, block->block_len); - uint64_t addr = dev->file_addr + len - 1; - if ((uint32_t)(addr >> 32) != dcr->EndFile || (uint32_t)addr > dcr->EndBlock) { - dcr->EndBlock = (uint32_t)addr; - dcr->EndFile = (uint32_t)(addr >> 32); + uint64_t addr = dev->get_full_addr() + len - 1; + if (dev->is_indexed()) { + if (addr > dcr->EndAddr) { + dcr->EndAddr = addr; + } } - dev->block_num = dev->EndBlock = (uint32_t)addr; - dev->file = dev->EndFile = (uint32_t)(addr >> 32); + dev->EndAddr = addr; + } + if (dev->is_indexed()) { + dcr->VolMediaId = dev->VolCatInfo.VolMediaId; } - dcr->VolMediaId = dev->VolCatInfo.VolMediaId; dev->file_addr += block->read_len; dev->file_size += block->read_len; + dev->usage += block->read_len; /* update usage counter */ /* * If we read a short block on disk, diff --git a/bacula/src/stored/block.h b/bacula/src/stored/block.h index 2217662205..1e126823db 100644 --- a/bacula/src/stored/block.h +++ b/bacula/src/stored/block.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -20,7 +20,6 @@ * Block definitions for Bacula media data format. * * Kern Sibbald, MM - * */ @@ -29,6 +28,9 @@ #define MAX_BLOCK_LENGTH 20000000 /* this is a sort of sanity check */ #define DEFAULT_BLOCK_SIZE (512 * 126) /* 64,512 N.B. do not use 65,636 here */ +#define MIN_DEDUP_BLOCK_SIZE (512 * 2) /* Minimum block (bucket) size */ + +#define DEDUP_BLOCK_SIZE (512 * 128) /* For now use a fixed dedup block size */ /* Block Header definitions. */ #define BLKHDR1_ID "BB01" @@ -40,6 +42,7 @@ #define WRITE_BLKHDR_ID BLKHDR2_ID #define WRITE_BLKHDR_LENGTH BLKHDR2_LENGTH +#define WRITE_ADATA_BLKHDR_LENGTH (6*sizeof(int32_t)+sizeof(uint64_t)) #define BLOCK_VER 2 /* Record header definitions */ @@ -53,6 +56,16 @@ #define RECHDR2_LENGTH (3*sizeof(int32_t)) #define WRITE_RECHDR_LENGTH RECHDR2_LENGTH +/* + * An adata record header includes: + * int32_t FileIndex + * int32_t Stream STREAM_ADATA_RECORD_HEADER + * uint32_t data_length + * uint32_t block length (binbuf to that point in time) + * int32_t Stream (original stream) + */ +#define WRITE_ADATA_RECHDR_LENGTH (5*sizeof(int32_t)) + /* Tape label and version definitions */ #define BaculaId "Bacula 1.0 immortal\n" #define OldBaculaId "Bacula 0.9 mortal\n" @@ -60,6 +73,17 @@ #define OldCompatibleBaculaTapeVersion1 10 #define OldCompatibleBaculaTapeVersion2 9 +#define BaculaMetaDataId "Bacula 1.0 Metadata\n" +#define BaculaAlignedDataId "Bacula 1.0 Aligned Data\n" +#define BaculaMetaDataVersion 10000 +#define BaculaAlignedDataVersion 20000 + +#define BaculaDedupMetaDataId "Bacula 1.0 Dedup Metadata\n" +#define BaculaDedupMetaDataVersion 30000 + +#define BaculaS3CloudId "Bacula 1.0 S3 Cloud Data\n" +#define BaculaS3CloudVersion 40000 + /* * This is the Media structure for a block header * Note, when written, it is serialized. @@ -80,6 +104,17 @@ uint32_t VolSessionId; uint32_t VolSessionTime; + * for an adata block header (in ameta file), we have + 32 bytes + + uint32_t BlockNumber; + int32_t Stream; STREAM_ADATA_BLOCK_HEADER + uint32_t block_len; + uint32_t CheckSum; + uint32_t VolSessionId; + uint32_t VolSessionTime; + uint64_t BlockAddr; + */ class DEVICE; /* for forward reference */ @@ -101,12 +136,10 @@ struct DEV_BLOCK { * For reads, it is remaining bytes not yet read. */ uint64_t BlockAddr; /* Block address */ - uint32_t File; /* Block address on volume */ - uint32_t Block; /* Block address on volume */ uint32_t binbuf; /* bytes in buffer */ uint32_t block_len; /* length of current block read */ uint32_t buf_len; /* max/default block length */ - uint32_t reclen; /* Last record length put in block */ + uint32_t reclen; /* Last record length put in adata block */ uint32_t BlockNumber; /* sequential Bacula block number */ uint32_t read_len; /* bytes read into buffer, if zero, block empty */ uint32_t VolSessionId; /* */ @@ -125,7 +158,8 @@ struct DEV_BLOCK { int32_t LastIndex; /* last index this block */ int32_t rechdr_items; /* number of items in rechdr queue */ char *bufp; /* pointer into buffer */ - char ser_buf[BLKHDR2_LENGTH]; /* Serial buffer for data */ + char ser_buf[BLKHDR2_LENGTH]; /* Serial buffer for adata */ + POOLMEM *rechdr_queue; /* record header queue */ POOLMEM *buf; /* actual data buffer */ }; diff --git a/bacula/src/stored/block_util.c b/bacula/src/stored/block_util.c index 657e9b1ff3..5ca251ad09 100644 --- a/bacula/src/stored/block_util.c +++ b/bacula/src/stored/block_util.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,14 +21,13 @@ * block_util.c -- tape block utility functions * * Kern Sibbald, split from block.c March MMXII - * */ #include "bacula.h" #include "stored.h" -static const int dbgel = 160; +static const int dbglvl = 160; #ifdef DEBUG_BLOCK_CHECKSUM static const bool debug_block_checksum = true; @@ -46,10 +45,11 @@ static const bool no_tape_write_test = false; * Dump the block header, then walk through * the block printing out the record headers. */ -void dump_block(DEV_BLOCK *b, const char *msg) +void dump_block(DEVICE *dev, DEV_BLOCK *b, const char *msg, bool force) { ser_declare; char *p; + char *bufp; char Id[BLKHDR_ID_LENGTH+1]; uint32_t CheckSum, BlockCheckSum; uint32_t block_len, reclen; @@ -60,9 +60,19 @@ void dump_block(DEV_BLOCK *b, const char *msg) int bhl, rhl; char buf1[100], buf2[100]; - if ((debug_level & ~DT_ALL) < 250) { + if (!force && ((debug_level & ~DT_ALL) < 250)) { + return; + } + if (b->adata) { + Dmsg0(20, "Dump block: adata=1 cannot dump.\n"); return; } + bufp = b->bufp; + if (dev) { + if (dev->can_read()) { + bufp = b->buf + b->block_len; + } + } unser_begin(b->buf, BLKHDR1_LENGTH); unser_uint32(CheckSum); unser_uint32(block_len); @@ -82,20 +92,19 @@ void dump_block(DEV_BLOCK *b, const char *msg) } if (block_len > 4000000 || block_len < BLKHDR_CS_LENGTH) { - Dmsg4(20, "!!!Dump block %s %p blocksize too %s %lu\n", - msg, b, + Dmsg3(20, "Will not dump blocksize too %s %lu msg: %s\n", (block_len < BLKHDR_CS_LENGTH)?"small":"big", - block_len); + block_len, msg); return; } BlockCheckSum = bcrc32((uint8_t *)b->buf+BLKHDR_CS_LENGTH, block_len-BLKHDR_CS_LENGTH); - Pmsg6(000, _("Dump block %s %p: size=%d BlkNum=%d\n" -" Hdrcksum=%x cksum=%x\n"), - msg, b, block_len, BlockNumber, CheckSum, BlockCheckSum); + Pmsg7(000, _("Dump block %s %p: adata=%d size=%d BlkNum=%d\n" +" Hdrcksum=%x cksum=%x\n"), + msg, b, b->adata, block_len, BlockNumber, CheckSum, BlockCheckSum); p = b->buf + bhl; - while (p < b->bufp) { + while (p < bufp) { unser_begin(p, WRITE_RECHDR_LENGTH); if (rhl == RECHDR1_LENGTH) { unser_uint32(VolSessionId); @@ -104,8 +113,18 @@ void dump_block(DEV_BLOCK *b, const char *msg) unser_int32(FileIndex); unser_int32(Stream); unser_uint32(data_len); - reclen = 0; - p += data_len + rhl; + if (Stream == STREAM_ADATA_BLOCK_HEADER) { + reclen = 0; + p += WRITE_ADATA_BLKHDR_LENGTH; + } else if (Stream == STREAM_ADATA_RECORD_HEADER || + Stream == -STREAM_ADATA_RECORD_HEADER) { + unser_uint32(reclen); + unser_int32(Stream); + p += WRITE_ADATA_RECHDR_LENGTH; + } else { + reclen = 0; + p += data_len + rhl; + } Pmsg6(000, _(" Rec: VId=%u VT=%u FI=%s Strm=%s len=%d reclen=%d\n"), VolSessionId, VolSessionTime, FI_to_ascii(buf1, FileIndex), stream_to_ascii(buf2, Stream, FileIndex), data_len, reclen); @@ -117,7 +136,12 @@ void dump_block(DEV_BLOCK *b, const char *msg) * We pass device so that the block can inherit the * min and max block sizes. */ -DEV_BLOCK *new_block(DEVICE *dev) +void DEVICE::new_dcr_blocks(DCR *dcr) +{ + dcr->block = dcr->ameta_block = new_block(dcr); +} + +DEV_BLOCK *DEVICE::new_block(DCR *dcr, int size) { DEV_BLOCK *block = (DEV_BLOCK *)get_memory(sizeof(DEV_BLOCK)); int len; @@ -125,22 +149,25 @@ DEV_BLOCK *new_block(DEVICE *dev) memset(block, 0, sizeof(DEV_BLOCK)); /* If the user has specified a max_block_size, use it as the default */ - if (dev->max_block_size == 0) { + if (max_block_size == 0) { len = DEFAULT_BLOCK_SIZE; } else { - len = dev->max_block_size; + len = max_block_size; + } + block->dev = this; + /* special size */ + if (size) { + len = size; } - block->dev = dev; - /* - * Round to multiple of block size + ensure that - * the data length is a multiple of the block size - */ block->buf_len = len; block->buf = get_memory(block->buf_len); + block->rechdr_queue = get_memory(block->buf_len); block->rechdr_items = 0; + Dmsg2(510, "Rechdr len=%d max_items=%d\n", sizeof_pool_memory(block->rechdr_queue), + sizeof_pool_memory(block->rechdr_queue)/WRITE_ADATA_RECHDR_LENGTH); empty_block(block); block->BlockVer = BLOCK_VER; /* default write version */ - Dmsg2(150, "New block len=%d block=%p\n", len, block); + Dmsg3(150, "New block adata=%d len=%d block=%p\n", block->adata, len, block); return block; } @@ -152,13 +179,48 @@ DEV_BLOCK *dup_block(DEV_BLOCK *eblock) { DEV_BLOCK *block = (DEV_BLOCK *)get_memory(sizeof(DEV_BLOCK)); int buf_len = sizeof_pool_memory(eblock->buf); + int rechdr_len = sizeof_pool_memory(eblock->rechdr_queue); memcpy(block, eblock, sizeof(DEV_BLOCK)); block->buf = get_memory(buf_len); memcpy(block->buf, eblock->buf, buf_len); + + block->rechdr_queue = get_memory(rechdr_len); + memcpy(block->rechdr_queue, eblock->rechdr_queue, rechdr_len); + + /* bufp might point inside buf */ + if (eblock->bufp && + eblock->bufp >= eblock->buf && + eblock->bufp < (eblock->buf + buf_len)) + { + block->bufp = (eblock->bufp - eblock->buf) + block->buf; + + } else { + block->bufp = NULL; + } return block; } +/* + * Flush block to disk + */ +bool DEVICE::flush_block(DCR *dcr) +{ + if (!is_block_empty(dcr->block)) { + Dmsg0(dbglvl, "=== wpath 53 flush_ameta\n"); + Dmsg4(190, "Call flush_ameta_block BlockAddr=%lld nbytes=%d adata=%d block=%x\n", + dcr->block->BlockAddr, dcr->block->binbuf, dcr->adata_block->adata, dcr->adata_block); + dump_block(dcr->dev, dcr->block, "Flush_ameta_block"); + if (dcr->jcr->is_canceled() || !dcr->write_block_to_device()) { + Dmsg0(dbglvl, "=== wpath 54 flush_ameta\n"); + Dmsg0(190, "Failed to write ameta block to device, return false.\n"); + return false; + } + empty_block(dcr->block); + } + return true; +} + /* * Only the first block checksum error was reported. @@ -172,12 +234,45 @@ void print_block_read_errors(JCR *jcr, DEV_BLOCK *block) } } +/* We had a problem on some solaris platforms with the CRC32 library, some + * 8.4.x jobs uses a bad crc32 algorithm. We just try one then the + * other to not create false problems + */ +uint32_t DCR::crc32(unsigned char *buf, int len, uint32_t expected_crc) +{ +#if defined(HAVE_SUN_OS) && defined(HAVE_LITTLE_ENDIAN) + uint32_t crc = 0; + if (crc32_type) { + crc = bcrc32_bad(buf, len); + + } else { + crc = bcrc32(buf, len); + } + if (expected_crc != crc) { + crc32_type = !crc32_type; /* Next time, do it well right away */ + + if (crc32_type) { + crc = bcrc32_bad(buf, len); + + } else { + crc = bcrc32(buf, len); + } + } + return crc; +#else + return bcrc32(buf, len); +#endif +} -void DCR::free_blocks() +void DEVICE::free_dcr_blocks(DCR *dcr) { - if (block) { - free_block(block); + if (dcr->block == dcr->ameta_block) { + dcr->ameta_block = NULL; /* do not free twice */ } + free_block(dcr->block); + dcr->block = NULL; + free_block(dcr->ameta_block); + dcr->ameta_block = NULL; } /* @@ -185,31 +280,48 @@ void DCR::free_blocks() */ void free_block(DEV_BLOCK *block) { - if (block && block->buf) { - Dmsg1(999, "free_block buffer %x\n", block->buf); - free_memory(block->buf); - Dmsg1(999, "free_block block %x\n", block); + if (block) { + Dmsg1(999, "free_block block=%p\n", block); + if (block->buf) { + free_memory(block->buf); + } + if (block->rechdr_queue) { + free_memory(block->rechdr_queue); + } + Dmsg1(999, "=== free_block block %p\n", block); free_memory((POOLMEM *)block); } } bool is_block_empty(DEV_BLOCK *block) { - return block->binbuf <= WRITE_BLKHDR_LENGTH; + if (block->adata) { + Dmsg1(200, "=== adata=1 binbuf=%d\n", block->binbuf); + return block->binbuf <= 0; + } else { + Dmsg1(200, "=== adata=0 binbuf=%d\n", block->binbuf-WRITE_BLKHDR_LENGTH); + return block->binbuf <= WRITE_BLKHDR_LENGTH; + } } /* Empty the block -- for writing */ void empty_block(DEV_BLOCK *block) { - block->binbuf = WRITE_BLKHDR_LENGTH; - Dmsg3(200, "empty len=%d block=%p set binbuf=%d\n", - block->buf_len, block, block->binbuf); + if (block->adata) { + block->binbuf = 0; + } else { + block->binbuf = WRITE_BLKHDR_LENGTH; + } + Dmsg3(250, "empty_block: adata=%d len=%d set binbuf=%d\n", + block->adata, block->buf_len, block->binbuf); block->bufp = block->buf + block->binbuf; block->read_len = 0; block->write_failed = false; block->block_read = false; block->needs_write = false; block->FirstIndex = block->LastIndex = 0; + block->RecNum = 0; + block->BlockAddr = 0; } /* @@ -223,25 +335,32 @@ uint32_t ser_block_header(DEV_BLOCK *block, bool do_checksum) uint32_t block_len = block->binbuf; block->CheckSum = 0; - Dmsg1(160, "block_header: block_len=%d\n", block_len); - ser_begin(block->buf, BLKHDR2_LENGTH); - ser_uint32(block->CheckSum); - ser_uint32(block_len); - ser_uint32(block->BlockNumber); - ser_bytes(WRITE_BLKHDR_ID, BLKHDR_ID_LENGTH); - if (BLOCK_VER >= 2) { - ser_uint32(block->VolSessionId); - ser_uint32(block->VolSessionTime); - } - - /* Checksum whole block except for the checksum */ - if (do_checksum) { - block->CheckSum = bcrc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH, - block_len-BLKHDR_CS_LENGTH); - } - Dmsg1(160, "ser_block_header: checksum=%x\n", block->CheckSum); - ser_begin(block->buf, BLKHDR2_LENGTH); - ser_uint32(block->CheckSum); /* now add checksum to block header */ + if (block->adata) { + /* Checksum whole block */ + if (do_checksum) { + block->CheckSum = bcrc32((uint8_t *)block->buf, block_len); + } + } else { + Dmsg1(160, "block_header: block_len=%d\n", block_len); + ser_begin(block->buf, BLKHDR2_LENGTH); + ser_uint32(block->CheckSum); + ser_uint32(block_len); + ser_uint32(block->BlockNumber); + ser_bytes(WRITE_BLKHDR_ID, BLKHDR_ID_LENGTH); + if (BLOCK_VER >= 2) { + ser_uint32(block->VolSessionId); + ser_uint32(block->VolSessionTime); + } + + /* Checksum whole block except for the checksum */ + if (do_checksum) { + block->CheckSum = bcrc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH, + block_len-BLKHDR_CS_LENGTH); + } + Dmsg2(160, "ser_block_header: adata=%d checksum=%x\n", block->adata, block->CheckSum); + ser_begin(block->buf, BLKHDR2_LENGTH); + ser_uint32(block->CheckSum); /* now add checksum to block header */ + } return block->CheckSum; } @@ -252,7 +371,7 @@ uint32_t ser_block_header(DEV_BLOCK *block, bool do_checksum) * Returns: false on failure (not a block) * true on success */ -bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block) +bool unser_block_header(DCR *dcr, DEVICE *dev, DEV_BLOCK *block) { ser_declare; char Id[BLKHDR_ID_LENGTH+1]; @@ -260,8 +379,32 @@ bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block) uint32_t block_len; uint32_t block_end; uint32_t BlockNumber; + JCR *jcr = dcr->jcr; int bhl; + if (block->adata) { + /* Checksum the whole block */ + if (block->block_len <= block->read_len && dev->do_checksum()) { + BlockCheckSum = dcr->crc32((uint8_t *)block->buf, block->block_len, block->CheckSum); + if (BlockCheckSum != block->CheckSum) { + dev->dev_errno = EIO; + Mmsg5(dev->errmsg, _("Volume data error at %lld!\n" + "Adata block checksum mismatch in block=%u len=%d: calc=%x blk=%x\n"), + block->BlockAddr, block->BlockNumber, + block->block_len, BlockCheckSum, block->CheckSum); + if (block->read_errors == 0 || verbose >= 2) { + Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); + dump_block(dev, block, "with checksum error"); + } + block->read_errors++; + if (!forge_on) { + return false; + } + } + } + return true; + } + if (block->no_header) { return true; } @@ -294,6 +437,8 @@ bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block) bhl = BLKHDR2_LENGTH; block->BlockVer = 2; block->bufp = block->buf + bhl; + //Dmsg5(100, "Read-blkhdr Block=%p adata=%d buf=%p bufp=%p off=%d\n", block, block->adata, + // block->buf, block->bufp, block->bufp-block->buf); if (strncmp(Id, BLKHDR2_ID, BLKHDR_ID_LENGTH) != 0) { dev->dev_errno = EIO; Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Wanted ID: \"%s\", got \"%s\". Buffer discarded.\n"), @@ -310,7 +455,7 @@ bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block) dev->file, dev->block_num, BLKHDR2_ID, Id); Dmsg1(50, "%s", dev->errmsg); if (block->read_errors == 0 || verbose >= 2) { - Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); + Jmsg(jcr, M_FATAL, 0, "%s", dev->errmsg); } block->read_errors++; unser_uint32(block->VolSessionId); @@ -338,14 +483,16 @@ bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block) block_end = block_len; } block->binbuf = block_end - bhl; - Dmsg2(200, "set block=%p binbuf=%d\n", block, block->binbuf); + Dmsg3(200, "set block=%p adata=%d binbuf=%d\n", block, block->adata, block->binbuf); block->block_len = block_len; block->BlockNumber = BlockNumber; Dmsg3(390, "Read binbuf = %d %d block_len=%d\n", block->binbuf, bhl, block_len); if (block_len <= block->read_len && dev->do_checksum()) { - BlockCheckSum = bcrc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH, - block_len-BLKHDR_CS_LENGTH); + BlockCheckSum = dcr->crc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH, + block_len-BLKHDR_CS_LENGTH, + block->CheckSum); + if (BlockCheckSum != block->CheckSum) { dev->dev_errno = EIO; Mmsg6(dev->errmsg, _("Volume data error at %u:%u!\n" @@ -354,7 +501,7 @@ bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block) block_len, BlockCheckSum, block->CheckSum); if (block->read_errors == 0 || verbose >= 2) { Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg); - dump_block(block, "with checksum error"); + dump_block(dev, block, "with checksum error"); } block->read_errors++; if (!forge_on) { @@ -380,8 +527,8 @@ uint32_t get_len_and_clear_block(DEV_BLOCK *block, DEVICE *dev, uint32_t &pad) if (wlen != block->buf_len) { Dmsg2(250, "binbuf=%d buf_len=%d\n", block->binbuf, block->buf_len); - /* Adjust write size to min/max for tapes only */ - if (dev->is_tape()) { + /* Adjust write size to min/max for tapes and aligned only */ + if (dev->is_tape() || block->adata) { /* check for fixed block size */ if (dev->min_block_size == dev->max_block_size) { wlen = block->buf_len; /* fixed block size already rounded */ @@ -393,14 +540,18 @@ uint32_t get_len_and_clear_block(DEV_BLOCK *block, DEVICE *dev, uint32_t &pad) wlen = ((wlen + TAPE_BSIZE - 1) / TAPE_BSIZE) * TAPE_BSIZE; } } + if (block->adata && dev->padding_size > 0) { + /* Write to next aligned boundry */ + wlen = ((wlen + dev->padding_size - 1) / dev->padding_size) * dev->padding_size; + } ASSERT(wlen <= block->buf_len); /* Clear from end of data to end of block */ if (wlen-block->binbuf > 0) { memset(block->bufp, 0, wlen-block->binbuf); /* clear garbage */ } pad = wlen - block->binbuf; /* padding or zeros written */ - Dmsg4(150, "Zero end blk: cleared=%d buf_len=%d wlen=%d binbuf=%d\n", - pad, block->buf_len, wlen, block->binbuf); + Dmsg5(150, "Zero end blk: adata=%d cleared=%d buf_len=%d wlen=%d binbuf=%d\n", + block->adata, pad, block->buf_len, wlen, block->binbuf); } else { pad = 0; } @@ -413,16 +564,22 @@ uint32_t get_len_and_clear_block(DEV_BLOCK *block, DEVICE *dev, uint32_t &pad) * reached, and if so, return true, otherwise * return false. */ -bool user_volume_size_reached(DCR *dcr, bool quiet) +bool is_user_volume_size_reached(DCR *dcr, bool quiet) { bool hit_max1, hit_max2; uint64_t size, max_size; - DEVICE *dev = dcr->dev; + DEVICE *dev = dcr->ameta_dev; char ed1[50]; bool rtn = false; - Enter(dbgel); - size = dev->VolCatInfo.VolCatBytes + dcr->block->binbuf; + Enter(dbglvl); + if (dev->is_aligned()) { + /* Note, we reserve space for one ameta and one adata block */ + size = dev->VolCatInfo.VolCatBytes + dcr->ameta_block->buf_len + + dcr->adata_block->buf_len; + } else { + size = dev->VolCatInfo.VolCatBytes + dcr->ameta_block->binbuf; + } /* Limit maximum Volume size to value specified by user */ hit_max1 = (dev->max_volume_size > 0) && (size >= dev->max_volume_size); hit_max2 = (dev->VolCatInfo.VolCatMaxBytes > 0) && @@ -445,8 +602,8 @@ bool user_volume_size_reached(DCR *dcr, bool quiet) dev->print_name(), dev->getVolCatName()); rtn = true; } - Dmsg1(dbgel, "Return from user_volume_size_reached=%d\n", rtn); - Leave(dbgel); + Dmsg1(dbglvl, "Return from is_user_volume_size_reached=%d\n", rtn); + Leave(dbglvl); return rtn; } @@ -458,6 +615,8 @@ void reread_last_block(DCR *dcr) bool ok = true; DEVICE *dev = dcr->dev; JCR *jcr = dcr->jcr; + DEV_BLOCK *ameta_block = dcr->ameta_block; + DEV_BLOCK *adata_block = dcr->adata_block; DEV_BLOCK *block = dcr->block; /* * If the device is a tape and it supports backspace record, @@ -497,9 +656,8 @@ void reread_last_block(DCR *dcr) */ } if (ok) { - DEV_BLOCK *lblock = new_block(dev); + dev->new_dcr_blocks(dcr); /* Note, this can destroy dev->errmsg */ - dcr->block = lblock; if (!dcr->read_block_from_dev(NO_BLOCK_NUMBER_CHECK)) { Jmsg(jcr, M_ERROR, 0, _("Re-read last block at EOT failed. ERR=%s"), dev->errmsg); @@ -508,23 +666,25 @@ void reread_last_block(DCR *dcr) * If we wrote block and the block numbers don't agree * we have a possible problem. */ - if (lblock->BlockNumber != dev->LastBlock) { - if (dev->LastBlock > (lblock->BlockNumber + 1)) { + if (dcr->block->BlockNumber != dev->LastBlock) { + if (dev->LastBlock > (dcr->block->BlockNumber + 1)) { Jmsg(jcr, M_FATAL, 0, _( "Re-read of last block: block numbers differ by more than one.\n" "Probable tape misconfiguration and data loss. Read block=%u Want block=%u.\n"), - lblock->BlockNumber, dev->LastBlock); + dcr->block->BlockNumber, dev->LastBlock); } else { Jmsg(jcr, M_ERROR, 0, _( "Re-read of last block OK, but block numbers differ. Read block=%u Want block=%u.\n"), - lblock->BlockNumber, dev->LastBlock); + dcr->block->BlockNumber, dev->LastBlock); } } else { Jmsg(jcr, M_INFO, 0, _("Re-read of last block succeeded.\n")); } } - free_block(lblock); + dev->free_dcr_blocks(dcr); + dcr->ameta_block = ameta_block; dcr->block = block; + dcr->adata_block = adata_block; } } #endif @@ -539,15 +699,27 @@ bool terminate_writing_volume(DCR *dcr) { DEVICE *dev = dcr->dev; bool ok = true; + bool was_adata = false; - Enter(dbgel); + Enter(dbglvl); if (dev->is_ateot()) { return ok; /* already been here return now */ } + /* Work with ameta device */ + if (dev->adata) { + dev->set_ateot(); /* no more writing this Volume */ + dcr->adata_block->write_failed = true; + dcr->set_ameta(); + dev = dcr->ameta_dev; + was_adata = true; + } + /* Create a JobMedia record to indicated end of medium */ dev->VolCatInfo.VolCatFiles = dev->get_file(); + dev->VolCatInfo.VolLastPartBytes = dev->part_size; + dev->VolCatInfo.VolCatParts = dev->part; if (!dir_create_jobmedia_record(dcr)) { Dmsg0(50, "Error from create JobMedia\n"); dev->dev_errno = EIO; @@ -557,19 +729,20 @@ bool terminate_writing_volume(DCR *dcr) ok = false; } flush_jobmedia_queue(dcr->jcr); + bstrncpy(dev->LoadedVolName, dev->VolCatInfo.VolCatName, sizeof(dev->LoadedVolName)); dcr->block->write_failed = true; - if (dev->can_append() && !dev->weof(1)) { /* end the tape */ + if (dev->can_append() && !dev->weof(dcr, 1)) { /* end the tape */ dev->VolCatInfo.VolCatErrors++; - Jmsg(dcr->jcr, M_ERROR, 0, _("Error writing final EOF to tape. This Volume may not be readable.\n" - "%s"), dev->errmsg); + Jmsg(dcr->jcr, M_ERROR, 0, _("Error writing final EOF to tape. Volume %s may not be readable.\n" + "%s"), dev->VolCatInfo.VolCatName, dev->errmsg); ok = false; Dmsg0(50, "Error writing final EOF to volume.\n"); } if (ok) { - ok = write_ansi_ibm_labels(dcr, ANSI_EOV_LABEL, dev->VolHdr.VolumeName); + ok = dev->end_of_volume(dcr); } - Dmsg2(100, "Set VolCatStatus Full size=%lld vol=%s\n", + Dmsg3(100, "Set VolCatStatus Full adata=%d size=%lld vol=%s\n", dev->adata, dev->VolCatInfo.VolCatBytes, dev->VolCatInfo.VolCatName); /* If still in append mode mark volume Full */ @@ -590,17 +763,22 @@ bool terminate_writing_volume(DCR *dcr) /* Set new file/block parameters for current dcr */ set_new_file_parameters(dcr); - if (ok && dev->has_cap(CAP_TWOEOF) && dev->can_append() && !dev->weof(1)) { /* end the tape */ + if (ok && dev->has_cap(CAP_TWOEOF) && dev->can_append() && !dev->weof(dcr, 1)) { /* end the tape */ dev->VolCatInfo.VolCatErrors++; /* This may not be fatal since we already wrote an EOF */ - Jmsg(dcr->jcr, M_ERROR, 0, "%s", dev->errmsg); + if (dev->errmsg[0]) { + Jmsg(dcr->jcr, M_ERROR, 0, "%s", dev->errmsg); + } Dmsg0(50, "Writing second EOF failed.\n"); } dev->set_ateot(); /* no more writing this tape */ Dmsg2(150, "Leave terminate_writing_volume=%s -- %s\n", dev->getVolCatName(), ok?"OK":"ERROR"); - Leave(dbgel); + if (was_adata) { + dcr->set_adata(); + } + Leave(dbglvl); return ok; } @@ -620,6 +798,11 @@ bool check_for_newvol_or_newfile(DCR *dcr) return false; } /* If we wrote on Volume create a last jobmedia record for this job */ + if (!dcr->VolFirstIndex) { + Dmsg7(100, "Skip JobMedia Vol=%s wrote=%d MediaId=%lld FI=%lu LI=%lu StartAddr=%lld EndAddr=%lld\n", + dcr->VolumeName, dcr->WroteVol, dcr->VolMediaId, + dcr->VolFirstIndex, dcr->VolLastIndex, dcr->StartAddr, dcr->EndAddr); + } if (dcr->VolFirstIndex && !dir_create_jobmedia_record(dcr)) { dcr->dev->dev_errno = EIO; Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"), @@ -629,6 +812,7 @@ bool check_for_newvol_or_newfile(DCR *dcr) return false; } if (dcr->NewVol) { + Dmsg0(250, "Process NewVol\n"); flush_jobmedia_queue(jcr); /* Note, setting a new volume also handles any pending new file */ set_new_volume_parameters(dcr); @@ -661,6 +845,8 @@ bool do_new_file_bookkeeping(DCR *dcr) return false; } dev->VolCatInfo.VolCatFiles = dev->get_file(); + dev->VolCatInfo.VolLastPartBytes = dev->part_size; + dev->VolCatInfo.VolCatParts = dev->part; if (!dir_update_volume_info(dcr, false, false)) { Dmsg0(50, "Error from update_vol_info.\n"); Dmsg0(40, "Call terminate_writing_volume\n"); diff --git a/bacula/src/stored/bls.c b/bacula/src/stored/bls.c index ced369248a..0c89b37c23 100644 --- a/bacula/src/stored/bls.c +++ b/bacula/src/stored/bls.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,12 +21,12 @@ * Dumb program to do an "ls" of a Bacula 1.0 mortal file. * * Kern Sibbald, MM - * */ #include "bacula.h" #include "stored.h" #include "findlib/find.h" +#include "lib/cmd_parser.h" extern bool parse_sd_config(CONFIG *config, const char *configfile, int exit_code); @@ -52,10 +52,6 @@ static CONFIG *config; void *start_heap; #define CONFIG_FILE "bacula-sd.conf" char *configfile = NULL; -STORES *me = NULL; /* our Global resource */ -bool forge_on = false; -pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; bool detect_errors = false; int errors = 0; @@ -69,21 +65,21 @@ static void usage() PROG_COPYRIGHT "\n%sVersion: %s (%s)\n\n" "Usage: bls [options] \n" -" -b specify a bootstrap file\n" -" -c specify a Storage configuration file\n" -" -d set debug level to \n" -" -dt print timestamp in debug output\n" -" -e exclude list\n" -" -i include list\n" -" -j list jobs\n" -" -k list blocks\n" -" (no j or k option) list saved files\n" -" -L dump label\n" -" -p proceed inspite of errors\n" -" -v be verbose\n" -" -V specify Volume names (separated by |)\n" -" -E Check records to detect errors\n" -" -? print this message\n\n"), 2000, "", VERSION, BDATE); +" -b specify a bootstrap file\n" +" -c specify a Storage configuration file\n" +" -d set debug level to \n" +" -dt print timestamp in debug output\n" +" -e exclude list\n" +" -i include list\n" +" -j list jobs\n" +" -k list blocks\n" +" (no j or k option) list saved files\n" +" -L dump label\n" +" -p proceed inspite of errors\n" +" -V specify Volume names (separated by |)\n" +" -E Check records to detect errors\n" +" -v be verbose\n" +" -? print this message\n\n"), 2000, "", VERSION, BDATE); exit(1); } @@ -96,7 +92,9 @@ int main (int argc, char *argv[]) char *VolumeName= NULL; char *bsrName = NULL; bool ignore_label_errors = false; + BtoolsAskDirHandler askdir_handler; + init_askdir_handler(&askdir_handler); setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); @@ -111,7 +109,7 @@ int main (int argc, char *argv[]) ff = init_find_files(); - while ((ch = getopt(argc, argv, "b:c:d:e:i:jkLpvV:?E")) != -1) { + while ((ch = getopt(argc, argv, "b:c:d:e:i:jkLpvV:?EDF:")) != -1) { switch (ch) { case 'b': bsrName = optarg; @@ -132,10 +130,18 @@ int main (int argc, char *argv[]) if (*optarg == 't') { dbg_timestamp = true; } else { + char *p; + /* We probably find a tag list -d 10,sql,bvfs */ + if ((p = strchr(optarg, ',')) != NULL) { + *p = 0; + } debug_level = atoi(optarg); if (debug_level <= 0) { debug_level = 1; } + if (p) { + debug_parse_tags(p+1, &debug_level_tags); + } } break; @@ -212,7 +218,7 @@ int main (int argc, char *argv[]) configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); + config = New(CONFIG()); parse_sd_config(config, configfile, M_ERROR_TERM); setup_me(); load_sd_plugins(me->plugin_directory); @@ -268,14 +274,13 @@ int main (int argc, char *argv[]) return 0; } - static void do_close(JCR *jcr) { release_device(jcr->dcr); free_attr(attr); free_record(rec); free_jcr(jcr); - dev->term(); + dev->term(NULL); } @@ -327,16 +332,19 @@ static void do_blocks(char *infname) block->VolSessionId, block->VolSessionTime); if (verbose == 1) { read_record_from_block(dcr, rec); - Pmsg9(-1, _("File:blk=%u:%u blk_num=%u blen=%u First rec FI=%s SessId=%u SessTim=%u Strm=%s rlen=%d\n"), - dev->file, dev->block_num, + Pmsg8(-1, "Addr=%llu blk_num=%u blen=%u First rec FI=%s SessId=%u SessTim=%u Strm=%s rlen=%d\n", + dev->get_full_addr(), block->BlockNumber, block->block_len, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, rec->VolSessionTime, - stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len); + stream_to_ascii_ex(buf2, rec->Stream, rec->FileIndex), rec->data_len); rec->remainder = 0; - } else if (verbose > 1) { - dump_block(block, ""); + } else if (verbose > 1) { /* detailed block dump */ + Pmsg5(-1, "Blk=%u blen=%u bVer=%d SessId=%u SessTim=%u\n", + block->BlockNumber, block->block_len, block->BlockVer, + block->VolSessionId, block->VolSessionTime); + dump_block(dcr->dev, block, "", true); } else { - printf(_("Block: %d size=%d\n"), block->BlockNumber, block->block_len); + printf("Block: %d size=%d\n", block->BlockNumber, block->block_len); } } @@ -367,7 +375,7 @@ static void do_jobs(char *infname) static void do_ls(char *infname) { if (dump_label) { - dump_volume_label(dev); + dev->dump_volume_label(); return; } if (!read_records(dcr, record_cb, mount_next_read_volume)) { @@ -376,6 +384,7 @@ static void do_ls(char *infname) printf("%u files found.\n", num_files); } + /* * Called here for each record from read_records() */ @@ -385,13 +394,6 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) dump_label_record(dcr->dev, rec, verbose, false); return true; } - if (verbose) { - char buf1[100], buf2[100]; - Pmsg6(000, "Record: FI=%s SessId=%d Strm=%s len=%u remlen=%d data_len=%d\n", - FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, - stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_bytes, rec->remlen, - rec->data_len); - } /* File Attributes stream */ if (rec->maskedStream == STREAM_UNIX_ATTRIBUTES || @@ -417,13 +419,17 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) print_ls_output(jcr, attr); num_files++; } - } else if (rec->Stream == STREAM_PLUGIN_NAME) { + } else if (rec->maskedStream == STREAM_PLUGIN_NAME) { char data[100]; int len = MIN(rec->data_len+1, sizeof(data)); bstrncpy(data, rec->data, len); Dmsg1(100, "Plugin data: %s\n", data); - } else if (rec->Stream == STREAM_RESTORE_OBJECT) { + } else if (rec->maskedStream == STREAM_RESTORE_OBJECT) { Dmsg0(100, "Restore Object record\n"); + } else if (rec->maskedStream == STREAM_ADATA_BLOCK_HEADER) { + Dmsg0(000, "Adata block header\n"); + } else if (rec->maskedStream == STREAM_ADATA_RECORD_HEADER) { + Dmsg0(000, "Adata record header\n"); } return true; @@ -476,33 +482,3 @@ static void get_session_record(DEVICE *dev, DEV_RECORD *rec, SESSION_LABEL *sess rtype, rec->VolSessionId, rec->VolSessionTime, rec->Stream, rec->data_len); } } - - -/* Dummies to replace askdir.c */ -bool dir_find_next_appendable_volume(DCR *dcr) { return 1;} -bool dir_update_volume_info(DCR *dcr, bool relabel, bool update_LastWritten) { return 1; } -bool dir_create_jobmedia_record(DCR *dcr, bool zero) { return 1; } -bool flush_jobmedia_queue(JCR *jcr) { return true; } -bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { return 1; } -bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { return 1;} -bool dir_send_job_status(JCR *jcr) {return 1;} -int generate_job_event(JCR *jcr, const char *event) { return 1; } - - -bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /*writing*/) -{ - DEVICE *dev = dcr->dev; - fprintf(stderr, _("Mount Volume \"%s\" on device %s and press return when ready: "), - dcr->VolumeName, dev->print_name()); - dev->close(); - getchar(); - return true; -} - -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) -{ - Dmsg0(100, "Fake dir_get_volume_info\n"); - dcr->setVolCatName(dcr->VolumeName); - Dmsg2(500, "Vol=%s VolType=%d\n", dcr->getVolCatName(), dcr->VolCatInfo.VolCatType); - return 1; -} diff --git a/bacula/src/stored/bscan.c b/bacula/src/stored/bscan.c index e9c6678107..f06d5f51ae 100644 --- a/bacula/src/stored/bscan.c +++ b/bacula/src/stored/bscan.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -23,7 +23,6 @@ * with the tape. * * Kern E. Sibbald, December 2001 - * */ #include "bacula.h" @@ -100,10 +99,6 @@ static CONFIG *config; void *start_heap; char *configfile = NULL; -STORES *me = NULL; /* our Global resource */ -bool forge_on = false; /* proceed inspite of I/O errors */ -pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; static void usage() { @@ -121,9 +116,6 @@ PROG_COPYRIGHT " -u specify database user name (default bacula)\n" " -P specify database password (default none)\n" " -h specify database host (default NULL)\n" -" -k path name to the key file (default NULL)\n" -" -e path name to the certificate file (default NULL)\n" -" -a path name to the CA certificate file (default NULL)\n" " -t specify database port (default 0)\n" " -p proceed inspite of I/O errors\n" " -r list records\n" @@ -142,7 +134,9 @@ int main (int argc, char *argv[]) int ch; struct stat stat_buf; char *VolumeName = NULL; + BtoolsAskDirHandler askdir_handler; + init_askdir_handler(&askdir_handler); setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); @@ -154,7 +148,7 @@ int main (int argc, char *argv[]) OSDependentInit(); - while ((ch = getopt(argc, argv, "b:c:d:D:h:k:e:a:p:mn:pP:rsSt:u:vV:w:?")) != -1) { + while ((ch = getopt(argc, argv, "b:c:d:D:h:p:mn:pP:rsSt:u:vV:w:?")) != -1) { switch (ch) { case 'S' : showProgress = true; @@ -189,18 +183,6 @@ int main (int argc, char *argv[]) db_host = optarg; break; - case 'k': - db_ssl_key = optarg; - break; - - case 'e': - db_ssl_cert = optarg; - break; - - case 'a': - db_ssl_ca = optarg; - break; - case 't': db_port = atoi(optarg); break; @@ -263,7 +245,7 @@ int main (int argc, char *argv[]) configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); + config = New(CONFIG()); parse_sd_config(config, configfile, M_ERROR_TERM); setup_me(); load_sd_plugins(me->plugin_directory); @@ -333,13 +315,13 @@ int main (int argc, char *argv[]) } free_jcr(bjcr); - dev->term(); + dev->term(NULL); return 0; } /* - * We are at the end of reading a tape. Now, we simulate handling - * the end of writing a tape by wiffling through the attached + * We are at the end of reading a Volume. Now, we simulate handling + * the end of writing a Volume by wiffling through the attached * jcrs creating jobmedia records. */ static bool bscan_mount_next_read_volume(DCR *dcr) @@ -356,10 +338,8 @@ static bool bscan_mount_next_read_volume(DCR *dcr) if (verbose) { Pmsg1(000, _("Create JobMedia for Job %s\n"), mjcr->Job); } - mdcr->StartBlock = dcr->StartBlock; - mdcr->StartFile = dcr->StartFile; - mdcr->EndBlock = dcr->EndBlock; - mdcr->EndFile = dcr->EndFile; + mdcr->StartAddr = dcr->StartAddr; + mdcr->EndAddr = dcr->EndAddr; mdcr->VolMediaId = dcr->VolMediaId; mjcr->read_dcr->VolLastIndex = dcr->VolLastIndex; if( mjcr->bscan_insert_jobmedia_records ) { @@ -514,8 +494,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) /* Reset some DCR variables */ foreach_dlist(dcr, dev->attached_dcrs) { dcr->VolFirstIndex = dcr->FileIndex = 0; - dcr->StartBlock = dcr->EndBlock = 0; - dcr->StartFile = dcr->EndFile = 0; + dcr->StartAddr = dcr->EndAddr = 0; dcr->VolMediaId = 0; } @@ -664,8 +643,8 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) free_jcr(mjcr); } } - mr.VolFiles = rec->File; - mr.VolBlocks = rec->Block; + mr.VolFiles = (uint32_t)(rec->Addr >> 32); + mr.VolBlocks = (uint32_t)rec->Addr; mr.VolBytes += mr.VolBlocks * WRITE_BLKHDR_LENGTH; /* approx. */ mr.VolMounts++; update_media_record(db, &mr); @@ -711,12 +690,11 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) fr.FileId = 0; num_files++; if (verbose && (num_files & 0x7FFF) == 0) { - char ed1[30], ed2[30], ed3[30], ed4[30]; - Pmsg4(000, _("%s file records. At file:blk=%s:%s bytes=%s\n"), + char ed1[30], ed2[30], ed3[30]; + Pmsg3(000, _("%s file records. At addr=%s bytes=%s\n"), edit_uint64_with_commas(num_files, ed1), - edit_uint64_with_commas(rec->File, ed2), - edit_uint64_with_commas(rec->Block, ed3), - edit_uint64_with_commas(mr.VolBytes, ed4)); + edit_uint64_with_commas(rec->Addr, ed2), + edit_uint64_with_commas(mr.VolBytes, ed3)); } create_file_attributes_record(db, mjcr, attr->fname, attr->lname, attr->type, attr->attr, rec); @@ -1182,8 +1160,15 @@ static int update_job_record(BDB *db, JOB_DBR *jr, SESSION_LABEL *elabel, mjcr->end_time = jr->EndTime; jr->JobId = mjcr->JobId; - jr->JobStatus = elabel->JobStatus; - mjcr->JobStatus = elabel->JobStatus; + + /* The JobStatus can't be 0 */ + if (elabel->JobStatus == 0) { + Pmsg2(000, _("Could not find JobStatus for SessId=%d SessTime=%d in EOS record.\n"), + rec->VolSessionId, rec->VolSessionTime); + } + mjcr->JobStatus = jr->JobStatus = + elabel->JobStatus ? elabel->JobStatus : JS_ErrorTerminated; + jr->JobFiles = elabel->JobFiles; if (jr->JobFiles > 0) { /* If we found files, force PurgedFiles */ jr->PurgedFiles = 0; @@ -1273,8 +1258,7 @@ static int create_jobmedia_record(BDB *db, JCR *mjcr) JOBMEDIA_DBR jmr; DCR *dcr = mjcr->read_dcr; - dcr->EndBlock = dev->EndBlock; - dcr->EndFile = dev->EndFile; + dcr->EndAddr = dev->EndAddr; dcr->VolMediaId = dev->VolCatInfo.VolMediaId; memset(&jmr, 0, sizeof(jmr)); @@ -1282,12 +1266,10 @@ static int create_jobmedia_record(BDB *db, JCR *mjcr) jmr.MediaId = mr.MediaId; jmr.FirstIndex = dcr->VolFirstIndex; jmr.LastIndex = dcr->VolLastIndex; - jmr.StartFile = dcr->StartFile; - jmr.EndFile = dcr->EndFile; - jmr.StartBlock = dcr->StartBlock; - jmr.EndBlock = dcr->EndBlock; - - + jmr.StartBlock = (uint32_t)dcr->StartAddr; + jmr.StartFile = (uint32_t)(dcr->StartAddr >> 32); + jmr.EndBlock = (uint32_t)dcr->EndAddr; + jmr.EndFile = (uint32_t)(dcr->EndAddr >> 32); if (!update_db) { return 1; } @@ -1364,33 +1346,3 @@ static JCR *create_jcr(JOB_DBR *jr, DEV_RECORD *rec, uint32_t JobId) return jobjcr; } - -/* Dummies to replace askdir.c */ -bool dir_find_next_appendable_volume(DCR *dcr) { return 1;} -bool dir_update_volume_info(DCR *dcr, bool relabel, bool update_LastWritten) { return 1; } -bool dir_create_jobmedia_record(DCR *dcr, bool zero) { return 1; } -bool flush_jobmedia_queue(JCR *jcr) { return true; } -bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { return 1; } -bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { return 1;} -bool dir_send_job_status(JCR *jcr) {return 1;} -int generate_job_event(JCR *jcr, const char *event) { return 1; } - -bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /*writing*/) -{ - DEVICE *dev = dcr->dev; - Dmsg0(20, "Enter dir_ask_sysop_to_mount_volume\n"); - /* Close device so user can use autochanger if desired */ - fprintf(stderr, _("Mount Volume \"%s\" on device %s and press return when ready: "), - dcr->VolumeName, dev->print_name()); - dev->close(); - getchar(); - return true; -} - -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) -{ - Dmsg0(100, "Fake dir_get_volume_info\n"); - dcr->setVolCatName(dcr->VolumeName); - Dmsg2(500, "Vol=%s VolType=%d\n", dcr->getVolCatName(), dcr->VolCatInfo.VolCatType); - return 1; -} diff --git a/bacula/src/stored/bsdjson.c b/bacula/src/stored/bsdjson.c new file mode 100644 index 0000000000..f24ab13ee7 --- /dev/null +++ b/bacula/src/stored/bsdjson.c @@ -0,0 +1,680 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula conf to json + * + * Kern Sibbald, MMXII + */ + +#include "bacula.h" +#include "stored.h" + +/* Imported functions */ +extern bool parse_sd_config(CONFIG *config, const char *configfile, int exit_code); + +/* Imported variables */ +#if defined(_MSC_VER) +extern "C" { // work around visual compiler mangling variables + extern URES res_all; +} +#else +extern URES res_all; +#endif +extern s_kw msg_types[]; +extern s_kw dev_types[]; +extern s_kw tapelabels[]; +extern s_kw cloud_drivers[]; +extern s_kw trunc_opts[]; +extern s_kw upload_opts[]; +extern s_kw proto_opts[]; +extern s_kw uri_opts[]; + +extern RES_TABLE resources[]; + +typedef struct +{ + bool do_list; + bool do_one; + bool do_only_data; + char *resource_type; + char *resource_name; + regex_t directive_reg; +} display_filter; + +/* Forward referenced functions */ +void terminate_stored(int sig); +static int check_resources(); +static void sendit(void *sock, const char *fmt, ...); +static void dump_json(display_filter *filter); + +#define CONFIG_FILE "bacula-sd.conf" /* Default config file */ + +/* Global variables exported */ +STORES *me = NULL; /* our Global resource */ + +char *configfile = NULL; + +/* Global static variables */ +static CONFIG *config; + + +static void usage() +{ + fprintf(stderr, _( +PROG_COPYRIGHT +"\n%sVersion: %s (%s)\n\n" +"Usage: bsdjson [options] [config_file]\n" +" -r get resource type \n" +" -n get resource \n" +" -l get only directives matching dirs (use with -r)\n" +" -D get only data\n" +" -c use as configuration file\n" +" -d set debug level to \n" +" -dt print timestamp in debug output\n" +" -t test - read config and exit\n" +" -v verbose user messages\n" +" -? print this message.\n" +"\n"), 2012, "", VERSION, BDATE); + + exit(1); +} + +/********************************************************************* + * + * Main Bacula Unix Storage Daemon + * + */ +#if defined(HAVE_WIN32) +#define main BaculaMain +#endif + +int main (int argc, char *argv[]) +{ + int ch; + bool test_config = false; + display_filter filter; + memset(&filter, 0, sizeof(filter)); + + setlocale(LC_ALL, ""); + bindtextdomain("bacula", LOCALEDIR); + textdomain("bacula"); + + my_name_is(argc, argv, "bacula-sd"); + init_msg(NULL, NULL); + + while ((ch = getopt(argc, argv, "Dc:d:tv?r:n:l:")) != -1) { + switch (ch) { + case 'D': + filter.do_only_data = true; + break; + + case 'l': + filter.do_list = true; + /* Might use something like -l '^(Name|Description)$' */ + filter.do_list = true; + if (regcomp(&filter.directive_reg, optarg, REG_EXTENDED) != 0) { + Jmsg((JCR *)NULL, M_ERROR_TERM, 0, + _("Please use valid -l argument: %s\n"), optarg); + } + break; + + case 'r': + filter.resource_type = optarg; + break; + + case 'n': + filter.resource_name = optarg; + break; + + case 'c': /* configuration file */ + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(optarg); + break; + + case 'd': /* debug level */ + if (*optarg == 't') { + dbg_timestamp = true; + } else { + debug_level = atoi(optarg); + if (debug_level <= 0) { + debug_level = 1; + } + } + break; + + case 't': + test_config = true; + break; + + case 'v': /* verbose */ + verbose++; + break; + + case '?': + default: + usage(); + break; + } + } + argc -= optind; + argv += optind; + + if (argc) { + if (configfile != NULL) { + free(configfile); + } + configfile = bstrdup(*argv); + argc--; + argv++; + } + + if (argc) { + usage(); + } + + if (filter.do_list && !filter.resource_type) { + usage(); + } + + if (filter.resource_type && filter.resource_name) { + filter.do_one = true; + } + + if (configfile == NULL || configfile[0] == 0) { + configfile = bstrdup(CONFIG_FILE); + } + + if (test_config && verbose > 0) { + char buf[1024]; + find_config_file(configfile, buf, sizeof(buf)); + sendit(NULL, "config_file=%s\n", buf); + } + + config = New(CONFIG()); + config->encode_password(false); + parse_sd_config(config, configfile, M_ERROR_TERM); + + if (!check_resources()) { + Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); + } + + if (test_config) { + terminate_stored(0); + } + + my_name_is(0, (char **)NULL, me->hdr.name); /* Set our real name */ + + dump_json(&filter); + + if (filter.do_list) { + regfree(&filter.directive_reg); + } + + terminate_stored(0); +} + +static void display_devtype(HPKT &hpkt) +{ + int i; + for (i=0; dev_types[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == dev_types[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + dev_types[i].name); + return; + } + } +} + +static void display_label(HPKT &hpkt) +{ + int i; + for (i=0; tapelabels[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == tapelabels[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + tapelabels[i].name); + return; + } + } +} + +static void display_cloud_driver(HPKT &hpkt) +{ + int i; + for (i=0; cloud_drivers[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == cloud_drivers[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + cloud_drivers[i].name); + return; + } + } +} + +static void display_protocol(HPKT &hpkt) +{ + int i; + for (i=0; proto_opts[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == proto_opts[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + proto_opts[i].name); + return; + } + } +} + +static void display_truncate_cache(HPKT &hpkt) +{ + int i; + for (i=0; trunc_opts[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == trunc_opts[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + trunc_opts[i].name); + return; + } + } +} + +static void display_uri_style(HPKT &hpkt) +{ + int i; + for (i=0; uri_opts[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == uri_opts[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + uri_opts[i].name); + return; + } + } +} + +static void display_upload(HPKT &hpkt) +{ + int i; + for (i=0; upload_opts[i].name; i++) { + if (*(int32_t *)(hpkt.ritem->value) == upload_opts[i].token) { + sendit(NULL, "\n \"%s\": \"%s\"", hpkt.ritem->name, + upload_opts[i].name); + return; + } + } +} + +/* + * Dump out all resources in json format. + * Note!!!! This routine must be in this file rather + * than in src/lib/parser_conf.c otherwise the pointers + * will be all messed up. + */ +static void dump_json(display_filter *filter) +{ + int resinx, item, directives, first_directive; + bool first_res; + RES_ITEM *items; + RES *res; + HPKT hpkt; + regmatch_t pmatch[32]; + STORES *me = (STORES *)GetNextRes(R_STORAGE, NULL); + + init_hpkt(hpkt); + + if (filter->do_only_data) { + sendit(NULL, "["); + + /* List resources and directives */ + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } + * or print a single item + */ + } else if (filter->do_one || filter->do_list) { + sendit(NULL, "{"); + + } else { + /* [ { "Device": { "Name": "aa",.. } }, { "Director": { "Name": "bb", ... } } ]*/ + sendit(NULL, "["); + } + + first_res = true; + /* Loop over all resource types */ + for (resinx=0; resources[resinx].name; resinx++) { + if (!resources[resinx].items) { + continue; /* skip dummy entries */ + } + + /* Skip this resource type */ + if (filter->resource_type && + strcasecmp(filter->resource_type, resources[resinx].name) != 0) { + continue; + } + + directives = 0; + /* Loop over all resources of this type */ + foreach_rblist(res, res_head[resinx]->res_list) { + hpkt.res = res; + items = resources[resinx].items; + if (!items) { + continue; + } + + /* Copy the resource into res_all */ + memcpy(&res_all, res, sizeof(res_all)); + + if (filter->resource_name) { + bool skip=true; + /* The Name should be at the first place, so this is not a real loop */ + for (item=0; items[item].name; item++) { + if (strcasecmp(items[item].name, "Name") == 0) { + if (strcasecmp(*(items[item].value), filter->resource_name) == 0) { + skip = false; + } + break; + } + } + if (skip) { /* The name doesn't match, so skip it */ + continue; + } + } + + if (first_res) { + sendit(NULL, "\n"); + } else { + sendit(NULL, ",\n"); + } + + if (filter->do_only_data) { + sendit(NULL, " {"); + + } else if (filter->do_one) { + /* Nothing to print */ + + /* When sending the list, the form is: + * { aa: { Name: aa, Description: aadesc...}, bb: { Name: bb + */ + } else if (filter->do_list) { + /* Search and display Name, should be the first item */ + for (item=0; items[item].name; item++) { + if (strcmp(items[item].name, "Name") == 0) { + sendit(NULL, "%s: {\n", quote_string(hpkt.edbuf2, *items[item].value)); + break; + } + } + } else { + /* Begin new resource */ + sendit(NULL, "{\n \"%s\": {", resources[resinx].name); + } + + first_res = false; + first_directive = 0; + directives = 0; + for (item=0; items[item].name; item++) { + /* Check user argument -l */ + if (filter->do_list && + regexec(&filter->directive_reg, + items[item].name, 32, pmatch, 0) != 0) + { + continue; + } + + hpkt.ritem = &items[item]; + if (bit_is_set(item, res_all.hdr.item_present)) { + if (first_directive++ > 0) printf(","); + if (display_global_item(hpkt)) { + /* Fall-through wanted */ + } else if (items[item].handler == store_maxblocksize) { + display_int32_pair(hpkt); + } else if (items[item].handler == store_devtype) { + display_devtype(hpkt); + } else if (items[item].handler == store_label) { + display_label(hpkt); + } else if (items[item].handler == store_cloud_driver) { + display_cloud_driver(hpkt); + } else if (items[item].handler == store_protocol) { + display_protocol(hpkt); + } else if (items[item].handler == store_uri_style) { + display_uri_style(hpkt); + } else if (items[item].handler == store_truncate) { + display_truncate_cache(hpkt); + } else if (items[item].handler == store_upload) { + display_upload(hpkt); + } else { + printf("\n \"%s\": \"null\"", items[item].name); + } + directives++; + } else { /* end if is present */ + /* For some directive, the bitmap is not set (like addresses) */ + if (me && strcmp(resources[resinx].name, "Storage") == 0) { + if (strcmp(items[item].name, "SdPort") == 0) { + if (get_first_port_host_order(me->sdaddrs) != items[item].default_value) { + if (first_directive++ > 0) sendit(NULL, ","); + sendit(NULL, "\n \"SdPort\": %d", + get_first_port_host_order(me->sdaddrs)); + } + } else if (me && strcmp(items[item].name, "SdAddress") == 0) { + char buf[500]; + get_first_address(me->sdaddrs, buf, sizeof(buf)); + if (strcmp(buf, "0.0.0.0") != 0) { + if (first_directive++ > 0) sendit(NULL, ","); + sendit(NULL, "\n \"SdAddress\": \"%s\"", buf); + } + } + } + } + if (items[item].flags & ITEM_LAST) { + display_last(hpkt); /* If last bit set always call to cleanup */ + } + } + + /* { "aa": { "Name": "aa",.. }, "bb": { "Name": "bb", ... } */ + if (filter->do_only_data || filter->do_list) { + sendit(NULL, "\n }"); /* Finish the Resource with a single } */ + + } else { + if (filter->do_one) { + /* don't print anything */ + + } else if (first_directive > 0) { + sendit(NULL, "\n }\n}"); /* end of resource */ + + } else { + sendit(NULL, "}\n }"); + } + } + + } /* End loop over all resources of this type */ + } /* End loop all resource types */ + + if (filter->do_only_data) { + sendit(NULL, "\n]\n"); + + /* In list context, we are dealing with a hash */ + } else if (filter->do_one || filter->do_list) { + sendit(NULL, "\n}\n"); + + } else { + sendit(NULL, "\n]\n"); + } + term_hpkt(hpkt); +} + + +/* Check Configuration file for necessary info */ +static int check_resources() +{ + bool OK = true; + bool tls_needed; + AUTOCHANGER *changer; + DEVRES *device; + + me = (STORES *)GetNextRes(R_STORAGE, NULL); + if (!me) { + Jmsg1(NULL, M_ERROR, 0, _("No Storage resource defined in %s. Cannot continue.\n"), + configfile); + OK = false; + } + + if (GetNextRes(R_STORAGE, (RES *)me) != NULL) { + Jmsg1(NULL, M_ERROR, 0, _("Only one Storage resource permitted in %s\n"), + configfile); + OK = false; + } + if (GetNextRes(R_DIRECTOR, NULL) == NULL) { + Jmsg1(NULL, M_ERROR, 0, _("No Director resource defined in %s. Cannot continue.\n"), + configfile); + OK = false; + } + if (GetNextRes(R_DEVICE, NULL) == NULL){ + Jmsg1(NULL, M_ERROR, 0, _("No Device resource defined in %s. Cannot continue.\n"), + configfile); + OK = false; + } + + if (!me->messages) { + me->messages = (MSGS *)GetNextRes(R_MSGS, NULL); + if (!me->messages) { + Jmsg1(NULL, M_ERROR, 0, _("No Messages resource defined in %s. Cannot continue.\n"), + configfile); + OK = false; + } + } + + if (!me->working_directory) { + Jmsg1(NULL, M_ERROR, 0, _("No Working Directory defined in %s. Cannot continue.\n"), + configfile); + OK = false; + } + + DIRRES *director; + STORES *store; + foreach_res(store, R_STORAGE) { + /* tls_require implies tls_enable */ + if (store->tls_require) { + if (have_tls) { + store->tls_enable = true; + } else { + Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); + OK = false; + continue; + } + } + + tls_needed = store->tls_enable || store->tls_authenticate; + + if (!store->tls_certfile && tls_needed) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Storage \"%s\" in %s.\n"), + store->hdr.name, configfile); + OK = false; + } + + if (!store->tls_keyfile && tls_needed) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Key\" file not defined for Storage \"%s\" in %s.\n"), + store->hdr.name, configfile); + OK = false; + } + + if ((!store->tls_ca_certfile && !store->tls_ca_certdir) && tls_needed && store->tls_verify_peer) { + Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Storage \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + store->hdr.name, configfile); + OK = false; + } + } + + foreach_res(director, R_DIRECTOR) { + /* tls_require implies tls_enable */ + if (director->tls_require) { + director->tls_enable = true; + } + + tls_needed = director->tls_enable || director->tls_authenticate; + + if (!director->tls_certfile && tls_needed) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Director \"%s\" in %s.\n"), + director->hdr.name, configfile); + OK = false; + } + + if (!director->tls_keyfile && tls_needed) { + Jmsg(NULL, M_FATAL, 0, _("\"TLS Key\" file not defined for Director \"%s\" in %s.\n"), + director->hdr.name, configfile); + OK = false; + } + + if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && tls_needed && director->tls_verify_peer) { + Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\"" + " or \"TLS CA Certificate Dir\" are defined for Director \"%s\" in %s." + " At least one CA certificate store is required" + " when using \"TLS Verify Peer\".\n"), + director->hdr.name, configfile); + OK = false; + } + } + + foreach_res(changer, R_AUTOCHANGER) { + foreach_alist(device, changer->device) { + device->cap_bits |= CAP_AUTOCHANGER; + } + } + + return OK; +} + +/* Clean up and then exit */ +void terminate_stored(int sig) +{ + static bool in_here = false; + + if (in_here) { /* prevent loops */ + bmicrosleep(2, 0); /* yield */ + exit(1); + } + in_here = true; + debug_level = 0; /* turn off any debug */ + + if (configfile) { + free(configfile); + configfile = NULL; + } + if (config) { + delete config; + config = NULL; + } + + if (debug_level > 10) { + print_memory_pool_stats(); + } + term_msg(); + free(res_head); + res_head = NULL; + close_memory_pool(); + + //sm_dump(false); /* dump orphaned buffers */ + exit(sig); +} + +static void sendit(void *sock, const char *fmt, ...) +{ + char buf[3000]; + va_list arg_ptr; + + va_start(arg_ptr, fmt); + bvsnprintf(buf, sizeof(buf), (char *)fmt, arg_ptr); + va_end(arg_ptr); + fputs(buf, stdout); + fflush(stdout); +} diff --git a/bacula/src/stored/btape.c b/bacula/src/stored/btape.c index 581752015f..f6e50c1a12 100644 --- a/bacula/src/stored/btape.c +++ b/bacula/src/stored/btape.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -27,7 +27,6 @@ * * Note, this program reads stored.conf, and will only * talk to devices that are configured. - * */ #include "bacula.h" @@ -48,10 +47,6 @@ int quit = 0; char buf[100000]; int bsize = TAPE_BSIZE; char VolName[MAX_NAME_LENGTH]; -STORES *me = NULL; /* our Global resource */ -bool forge_on = false; /* proceed inspite of I/O errors */ -pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; /* * If you change the format of the state file, @@ -143,6 +138,17 @@ static void usage(); static void terminate_btape(int sig); int get_cmd(const char *prompt); +class BtapeAskDirHandler: public BtoolsAskDirHandler +{ +public: + BtapeAskDirHandler() {} + ~BtapeAskDirHandler() {} + bool dir_find_next_appendable_volume(DCR *dcr); + bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /* writing */); + bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr); + bool dir_create_jobmedia_record(DCR *dcr, bool zero); +}; + /********************************************************************* * @@ -155,7 +161,9 @@ int main(int margc, char *margv[]) uint32_t x32, y32; uint64_t x64, y64; char buf[1000]; + BtapeAskDirHandler askdir_handler; + init_askdir_handler(&askdir_handler); setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); @@ -200,11 +208,14 @@ int main(int margc, char *margv[]) OSDependentInit(); - while ((ch = getopt(margc, margv, "b:c:d:psv?")) != -1) { + while ((ch = getopt(margc, margv, "b:w:c:d:psv?")) != -1) { switch (ch) { case 'b': /* bootstrap file */ bsr = parse_bsr(NULL, optarg); -// dump_bsr(bsr, true); + break; + + case 'w': + working_directory = optarg; break; case 'c': /* specify config file */ @@ -260,7 +271,7 @@ int main(int margc, char *margv[]) daemon_start_time = time(NULL); - config = new_config_parser(); + config = New(CONFIG()); parse_sd_config(config, configfile, M_ERROR_TERM); setup_me(); load_sd_plugins(me->plugin_directory); @@ -284,11 +295,6 @@ int main(int margc, char *margv[]) if (!dev) { exit(1); } - if (dev->is_dvd()) { - Pmsg0(000, _("btape does not work with DVD storage.\n")); - usage(); - exit(1); - } if (!dev->is_tape()) { Pmsg0(000, _("btape only works with tape storage.\n")); usage(); @@ -309,8 +315,6 @@ static void terminate_btape(int stat) { Dsm_check(200); - free_jcr(jcr); - jcr = NULL; if (args) { free_pool_memory(args); @@ -325,11 +329,13 @@ static void terminate_btape(int stat) free_bsr(bsr); } + free_jcr(jcr); + jcr = NULL; free_volume_lists(); if (dev) { - dev->term(); + dev->term(dcr); } if (configfile) { @@ -337,22 +343,22 @@ static void terminate_btape(int stat) } if (config) { - config->free_resources(); - free(config); + delete config; config = NULL; } if (chk_dbglvl(10)) print_memory_pool_stats(); - if (this_block) { - free_block(this_block); - this_block = NULL; - } + Dmsg1(900, "=== free_block %p\n", this_block); + free_block(this_block); + this_block = NULL; stop_watchdog(); term_msg(); term_last_jobs_list(); + free(res_head); + res_head = NULL; close_memory_pool(); /* free memory in pool */ lmgr_cleanup_main(); @@ -457,13 +463,14 @@ static void mix_buffer(fill_mode_t mode, char *data, uint32_t len) static bool open_the_device() { - DEV_BLOCK *block; bool ok = true; - block = new_block(dev); + if (!dcr->block) { + dev->new_dcr_blocks(dcr); + } dev->rLock(false); Dmsg1(200, "Opening device %s\n", dcr->VolumeName); - if (!dev->open(dcr, OPEN_READ_WRITE)) { + if (!dev->open_device(dcr, OPEN_READ_WRITE)) { Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), dev->print_errmsg()); ok = false; goto bail_out; @@ -473,7 +480,6 @@ static bool open_the_device() bail_out: dev->Unlock(); - free_block(block); return ok; } @@ -502,7 +508,7 @@ static void labelcmd() } } dev->rewind(dcr); - write_new_volume_label_to_dev(dcr, cmd, "Default", false,/*no relabel*/ true /* label dvd now */); + dev->write_volume_label(dcr, cmd, "Default", false,/*no relabel*/ true/* label now */); Pmsg1(-1, _("Wrote Volume label for volume \"%s\".\n"), cmd); } @@ -514,7 +520,7 @@ static void readlabelcmd() int64_t save_debug_level = debug_level; int stat; - stat = read_dev_volume_label(dcr); + stat = dev->read_dev_volume_label(dcr); switch (stat) { case VOL_NO_LABEL: Pmsg0(0, _("Volume has no label.\n")); @@ -546,7 +552,7 @@ static void readlabelcmd() } debug_level = 20; - dump_volume_label(dev); + dev->dump_volume_label(); debug_level = save_debug_level; } @@ -598,7 +604,7 @@ static void weofcmd() num = 1; } - if (!dev->weof(num)) { + if (!dev->weof(NULL, num)) { Pmsg1(0, _("Bad status from weof. ERR=%s\n"), dev->bstrerror()); return; } else { @@ -748,7 +754,7 @@ static void rectestcmd() Dsm_check(200); save_block = dcr->block; - dcr->block = new_block(dev); + dcr->block = dev->new_block(dcr); rec = new_record(); for (i=1; i<500000; i++) { @@ -766,7 +772,8 @@ static void rectestcmd() Dsm_check(200); } free_record(rec); - free_block(dcr->block); + Dmsg0(900, "=== free_blocks\n"); + dcr->dev->free_dcr_blocks(dcr); dcr->block = save_block; /* restore block to dcr */ Dsm_check(200); } @@ -1277,7 +1284,6 @@ static bool position_test() bool got_eof = false; Pmsg0(0, _("Block position test\n")); - block = dcr->block; empty_block(block); rec = new_record(); rec->data = check_pool_memory_size(rec->data, block->buf_len); @@ -1332,7 +1338,9 @@ static bool position_test() continue; } Pmsg2(-1, _("Reposition to file:block %d:%d\n"), file, blk); - if (!dev->reposition(dcr, file, blk)) { + uint64_t addr = file; + addr = (addr<<32) + blk; + if (!dev->reposition(dcr, addr)) { Pmsg0(0, _("Reposition error.\n")); goto bail_out; } @@ -1414,7 +1422,7 @@ static int append_test() if (dev->has_cap(CAP_TWOEOF)) { weofcmd(); } - dev->close(); /* release device */ + dev->close(dcr); /* release device */ if (!open_the_device()) { return -1; } @@ -1509,7 +1517,7 @@ try_again: if (loaded) { dcr->VolCatInfo.Slot = loaded; /* We are going to load a new tape, so close the device */ - dev->close(); + dev->close(dcr); Pmsg2(-1, _("3302 Issuing autochanger \"unload %d %d\" command.\n"), loaded, dev->drive_index); changer = edit_device_codes(dcr, changer, @@ -1534,7 +1542,7 @@ try_again: changer = edit_device_codes(dcr, changer, dcr->device->changer_command, "load"); Dmsg1(100, "Changer=%s\n", changer); - dev->close(); + dev->close(dcr); status = run_program(changer, timeout, results); if (status == 0) { Pmsg2(-1, _("3303 Autochanger \"load %d %d\" status is OK.\n"), @@ -1554,7 +1562,7 @@ try_again: * a failure. */ bmicrosleep(sleep_time, 0); - if (!dev->rewind(dcr) || !dev->weof(1)) { + if (!dev->rewind(dcr) || !dev->weof(dcr, 1)) { Pmsg1(0, _("Bad status from rewind. ERR=%s\n"), dev->bstrerror()); dev->clrerror(-1); Pmsg0(-1, _("\nThe test failed, probably because you need to put\n" @@ -1566,7 +1574,7 @@ try_again: Pmsg1(0, _("Rewound %s\n"), dev->print_name()); } - if (!dev->weof(1)) { + if (!dev->weof(dcr, 1)) { Pmsg1(0, _("Bad status from weof. ERR=%s\n"), dev->bstrerror()); goto bail_out; } else { @@ -1870,7 +1878,7 @@ static void fsrcmd() */ static void rbcmd() { - dev->open(dcr, OPEN_READ_ONLY); + dev->open_device(dcr, OPEN_READ_ONLY); dcr->read_block_from_dev(NO_BLOCK_NUMBER_CHECK); } @@ -1889,7 +1897,7 @@ static void wrcmd() Dsm_check(200); empty_block(block); if (verbose > 1) { - dump_block(block, "test"); + dump_block(dcr->dev, block, "test"); } i = block->buf_len - 100; @@ -2118,7 +2126,7 @@ static void scan_blocks() rec->remainder = 0; free_record(rec); } else if (verbose > 1) { - dump_block(block, ""); + dump_block(dcr->dev, block, ""); } } @@ -2214,7 +2222,9 @@ static void fillcmd() * subroutine. */ Dmsg0(100, "just before acquire_device\n"); + dcr->setVolCatName(dcr->VolumeName); if (!acquire_device_for_append(dcr)) { + Pmsg0(000, "Could not acquire_device_for_append()\n"); jcr->setJobStatus(JS_ErrorTerminated); exit_code = 1; return; @@ -2306,7 +2316,7 @@ static void fillcmd() Pmsg1(-1, _("%s Flush block, write EOF\n"), buf1); flush_block(block, 0); #ifdef needed_xxx - dev->weof(1); + dev->weof(dcr, 1); #endif } @@ -2361,9 +2371,8 @@ static void fillcmd() /* Save last block info for second tape */ last_block_num2 = last_block_num; last_file2 = last_file; - if (last_block2) { - free_block(last_block2); - } + Dmsg1(000, "=== free_block %p\n", last_block2); + free_block(last_block2); last_block2 = dup_block(last_block); } @@ -2426,9 +2435,9 @@ static void unfillcmd() int fd; exit_code = 0; - last_block1 = new_block(dev); - last_block2 = new_block(dev); - first_block = new_block(dev); + last_block1 = dev->new_block(dcr); + last_block2 = dev->new_block(dcr); + first_block = dev->new_block(dcr); sprintf(buf, "%s/btape.state", working_directory); fd = open(buf, O_RDONLY); if (fd >= 0) { @@ -2472,6 +2481,7 @@ static bool do_unfill() DEV_BLOCK *block = dcr->block; int autochanger; bool rc = false; + uint64_t addr; dumped = 0; VolBytes = 0; @@ -2486,10 +2496,9 @@ static bool do_unfill() time(&jcr->run_time); /* start counting time for rates */ stop = 0; file_index = 0; - if (last_block) { - free_block(last_block); - last_block = NULL; - } + Dmsg1(900, "=== free_block %p\n", last_block); + free_block(last_block); + last_block = NULL; last_block_num = last_block_num1; last_file = last_file1; last_block = last_block1; @@ -2497,7 +2506,7 @@ static bool do_unfill() free_restore_volume_list(jcr); jcr->bsr = NULL; bstrncpy(dcr->VolumeName, "TestVolume1|TestVolume2", sizeof(dcr->VolumeName)); - create_restore_volume_list(jcr); + create_restore_volume_list(jcr, true); if (jcr->VolList != NULL) { jcr->VolList->Slot = 1; if (jcr->VolList->next != NULL) { @@ -2511,18 +2520,18 @@ static bool do_unfill() /* Multiple Volume tape */ /* Close device so user can use autochanger if desired */ if (dev->has_cap(CAP_OFFLINEUNMOUNT)) { - dev->offline(); + dev->offline(dcr); } autochanger = autoload_device(dcr, 1, NULL); if (autochanger != 1) { Pmsg1(100, "Autochanger returned: %d\n", autochanger); - dev->close(); + dev->close(dcr); get_cmd(_("Mount first tape. Press enter when ready: ")); Pmsg0(000, "\n"); } } - dev->close(); + dev->close(dcr); dev->num_writers = 0; dcr->clear_writing(); if (!acquire_device_for_read(dcr)) { @@ -2545,7 +2554,9 @@ static bool do_unfill() read_records(dcr, quickie_cb, my_mount_next_read_volume); Pmsg4(-1, _("Reposition from %u:%u to %u:%u\n"), dev->file, dev->block_num, last_file, last_block_num); - if (!dev->reposition(dcr, last_file, last_block_num)) { + addr = last_file; + addr = (addr << 32) + last_block_num; + if (!dev->reposition(dcr, addr)) { Pmsg1(-1, _("Reposition error. ERR=%s\n"), dev->bstrerror()); goto bail_out; } @@ -2574,7 +2585,7 @@ static bool do_unfill() /* Multiple Volume tape */ /* Close device so user can use autochanger if desired */ if (dev->has_cap(CAP_OFFLINEUNMOUNT)) { - dev->offline(); + dev->offline(dcr); } set_volume_name("TestVolume2", 2); @@ -2582,7 +2593,7 @@ static bool do_unfill() autochanger = autoload_device(dcr, 1, NULL); if (autochanger != 1) { Pmsg1(100, "Autochanger returned: %d\n", autochanger); - dev->close(); + dev->close(dcr); get_cmd(_("Mount second tape. Press enter when ready: ")); Pmsg0(000, "\n"); } @@ -2598,7 +2609,8 @@ static bool do_unfill() * on the previous tape. */ Pmsg2(-1, _("Reposition from %u:%u to 0:1\n"), dev->file, dev->block_num); - if (!dev->reposition(dcr, 0, 1)) { + addr = 1; + if (!dev->reposition(dcr, addr)) { Pmsg1(-1, _("Reposition error. ERR=%s\n"), dev->bstrerror()); goto bail_out; } @@ -2614,7 +2626,9 @@ static bool do_unfill() /* Now find and compare the last block */ Pmsg4(-1, _("Reposition from %u:%u to %u:%u\n"), dev->file, dev->block_num, last_file, last_block_num); - if (!dev->reposition(dcr, last_file, last_block_num)) { + addr = last_file; + addr = (addr<<32) + last_block_num; + if (!dev->reposition(dcr, addr)) { Pmsg1(-1, _("Reposition error. ERR=%s\n"), dev->bstrerror()); goto bail_out; } @@ -2668,9 +2682,9 @@ static bool compare_blocks(DEV_BLOCK *last_block, DEV_BLOCK *block) continue; } Pmsg0(-1, "\n"); - dump_block(last_block, _("Last block written")); + dump_block(NULL, last_block, _("Last block written")); Pmsg0(-1, "\n"); - dump_block(block, _("Block read back")); + dump_block(NULL, block, _("Block read back")); Pmsg1(-1, _("\n\nThe blocks differ at byte %u\n"), p - last_block->buf); Pmsg0(-1, _("\n\n!!!! The last block written and the block\n" "that was read back differ. The test FAILED !!!!\n" @@ -2679,8 +2693,8 @@ static bool compare_blocks(DEV_BLOCK *last_block, DEV_BLOCK *block) return false; } if (verbose) { - dump_block(last_block, _("Last block written")); - dump_block(block, _("Block read back")); + dump_block(NULL, last_block, _("Last block written")); + dump_block(NULL, block, _("Block read back")); } return true; } @@ -2699,10 +2713,10 @@ static int flush_block(DEV_BLOCK *block, int dump) dev->rLock(false); if (!this_block) { - this_block = new_block(dev); + this_block = dev->new_block(dcr); } if (!last_block) { - last_block = new_block(dev); + last_block = dev->new_block(dcr); } /* Copy block */ this_file = dev->file; @@ -2724,9 +2738,9 @@ static int flush_block(DEV_BLOCK *block, int dump) if (verbose) { Pmsg3(000, _("Block not written: FileIndex=%u blk_block=%u Size=%u\n"), (unsigned)file_index, block->BlockNumber, block->block_len); - dump_block(last_block, _("Last block written")); + dump_block(dev, last_block, _("Last block written")); Pmsg0(-1, "\n"); - dump_block(block, _("Block not written")); + dump_block(dev, block, _("Block not written")); } if (stop == 0) { eot_block = block->BlockNumber; @@ -2962,6 +2976,7 @@ PROG_COPYRIGHT " -dt print timestamp in debug output\n" " -p proceed inspite of I/O errors\n" " -s turn off signals\n" +" -w set working directory to dir\n" " -v be verbose\n" " -? print this message.\n" "\n"), 2000, "", VERSION, BDATE); @@ -3003,38 +3018,19 @@ get_cmd(const char *prompt) return 0; } -/* Dummies to replace askdir.c */ -bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { return 1;} -bool dir_send_job_status(JCR *jcr) {return 1;} -bool flush_jobmedia_queue(JCR *jcr) { return true; } - -bool dir_update_volume_info(DCR *dcr, bool relabel, bool update_LastWritten) -{ - return 1; -} - - -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw writing) -{ - Dmsg0(20, "Enter dir_get_volume_info\n"); - dcr->setVolCatName(dcr->VolumeName); - return 1; -} - -bool dir_create_jobmedia_record(DCR *dcr, bool zero) +bool BtapeAskDirHandler::dir_create_jobmedia_record(DCR *dcr, bool zero) { dcr->WroteVol = false; return 1; } - -bool dir_find_next_appendable_volume(DCR *dcr) +bool BtapeAskDirHandler::dir_find_next_appendable_volume(DCR *dcr) { Dmsg1(20, "Enter dir_find_next_appendable_volume. stop=%d\n", stop); return dcr->VolumeName[0] != 0; } -bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /* writing */) +bool BtapeAskDirHandler::dir_ask_sysop_to_mount_volume(DCR *dcr, bool /* writing */) { DEVICE *dev = dcr->dev; Dmsg0(20, "Enter dir_ask_sysop_to_mount_volume\n"); @@ -3049,12 +3045,12 @@ bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool /* writing */) fprintf(stderr, _("Mount Volume \"%s\" on device %s and press return when ready: "), dcr->VolumeName, dev->print_name()); } - dev->close(); + dev->close(dcr); getchar(); return true; } -bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) +bool BtapeAskDirHandler::dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { int autochanger; DEVICE *dev = dcr->dev; @@ -3066,14 +3062,14 @@ bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) } /* Close device so user can use autochanger if desired */ if (dev->has_cap(CAP_OFFLINEUNMOUNT)) { - dev->offline(); + dev->offline(dcr); } autochanger = autoload_device(dcr, 1, NULL); if (autochanger != 1) { Pmsg1(100, "Autochanger returned: %d\n", autochanger); fprintf(stderr, _("Mount blank Volume on device %s and press return when ready: "), dev->print_name()); - dev->close(); + dev->close(dcr); getchar(); Pmsg0(000, "\n"); } @@ -3116,7 +3112,7 @@ static bool my_mount_next_read_volume(DCR *dcr) set_volume_name("TestVolume2", 2); - dev->close(); + dev->close(dcr); if (!acquire_device_for_read(dcr)) { Pmsg2(0, _("Cannot open Dev=%s, Vol=%s\n"), dev->print_name(), dcr->VolumeName); return false; diff --git a/bacula/src/stored/butil.c b/bacula/src/stored/butil.c index b380dc8fb4..c59289f6b2 100644 --- a/bacula/src/stored/butil.c +++ b/bacula/src/stored/butil.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -26,7 +26,6 @@ * Normally nothing in this file is called by the Storage * daemon because we interact more directly with the user * i.e. printf, ... - * */ #include "bacula.h" @@ -186,7 +185,7 @@ static DCR *setup_to_access_device(JCR *jcr, char *dev_name, } bstrncpy(dcr->dev_name, device->device_name, sizeof(dcr->dev_name)); - create_restore_volume_list(jcr); + create_restore_volume_list(jcr, true); if (!writing) { /* read only access? */ Dmsg0(100, "Acquire device for read\n"); diff --git a/bacula/src/stored/cloud_dev.c b/bacula/src/stored/cloud_dev.c new file mode 100644 index 0000000000..259218dd81 --- /dev/null +++ b/bacula/src/stored/cloud_dev.c @@ -0,0 +1,18 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ diff --git a/bacula/src/stored/cloud_dev.h b/bacula/src/stored/cloud_dev.h new file mode 100644 index 0000000000..b843ea1813 --- /dev/null +++ b/bacula/src/stored/cloud_dev.h @@ -0,0 +1,78 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Generic routines for writing Cloud Volumes + * + * Written by Kern Sibbald, May MMXVI + */ + +#ifndef _CLOUD_DEV_H_ +#define _CLOUD_DEV_H_ + +#include "bacula.h" +#include "stored.h" +#include "cloud_driver.h" + +class cloud_dev: public file_dev { +public: + int64_t obj_len; + int status; + +public: + cloud_dev(JCR *jcr, DEVRES *device); + ~cloud_dev(); + + cloud_driver *driver; + + /* DEVICE virtual interfaces that we redefine */ + boffset_t lseek(DCR *dcr, off_t offset, int whence); + bool rewind(DCR *dcr); + bool reposition(DCR *dcr, uint64_t raddr); + bool open_device(DCR *dcr, int omode); + bool open_next_part(DCR *dcr); + bool truncate(DCR *dcr); + int truncate_cache(DCR *dcr, const char *VolName, int64_t *size); + bool upload_cache(DCR *dcr, const char *VolName, POOLMEM *&err); + bool close(DCR *dcr); + bool update_pos(DCR *dcr); + bool is_eod_valid(DCR *dcr); + bool eod(DCR *dcr); + int read_dev_volume_label(DCR *dcr); + const char *print_type(); + DEVICE *get_dev(DCR *dcr); + uint32_t get_hi_addr(); + uint32_t get_low_addr(); + uint64_t get_full_addr(); + uint64_t get_full_addr(boffset_t addr); + char *print_addr(char *buf, int32_t buf_len); + char *print_addr(char *buf, int32_t maxlen, boffset_t addr); + bool do_size_checks(DCR *dcr, DEV_BLOCK *block); + bool write_volume_label(DCR *dcr, + const char *VolName, const char *PoolName, + bool relabel, bool no_prelabel); + bool rewrite_volume_label(DCR *dcr, bool recycle); + bool start_of_job(DCR *dcr); + bool end_of_job(DCR *dcr); + bool get_cloud_volumes_list(DCR* dcr, alist *volumes, POOLMEM *&err) { return driver->get_cloud_volumes_list(dcr, volumes, err); }; + bool get_cloud_volume_parts_list(DCR *dcr, const char *VolumeName, ilist *parts, POOLMEM *&err) { return driver->get_cloud_volume_parts_list(dcr, VolumeName, parts, err);}; + uint32_t get_cloud_upload_transfer_status(POOL_MEM &msg, bool verbose); + uint32_t get_cloud_download_transfer_status(POOL_MEM &msg, bool verbose); +}; + +#endif /* _CLOUD_DEV_H_ */ diff --git a/bacula/src/stored/cloud_driver.h b/bacula/src/stored/cloud_driver.h new file mode 100644 index 0000000000..66558b3d2e --- /dev/null +++ b/bacula/src/stored/cloud_driver.h @@ -0,0 +1,57 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Routines for writing Cloud drivers + * + * Written by Kern Sibbald, May MMXVI + */ + +#include "bacula.h" +#include "stored.h" + +#ifndef _CLOUD_DRIVER_H_ +#define _CLOUD_DRIVER_H_ + +#define NUM_UPLOAD_RETRIES 2 +class cloud_dev; + +enum { + C_S3_DRIVER = 1, + C_FILE_DRIVER = 2 +}; + +/* Abstract class cannot be instantiated */ +class cloud_driver: public SMARTALLOC { +public: + cloud_driver() : max_upload_retries(NUM_UPLOAD_RETRIES) {}; + virtual ~cloud_driver() {}; + + virtual bool truncate_cloud_volume(DCR *dcr, const char *VolumeName, ilist *trunc_parts, POOLMEM *&err) = 0; + virtual bool init(JCR *jcr, cloud_dev *dev, DEVRES *device) = 0; + virtual bool term(DCR *dcr) = 0; + virtual bool start_of_job(DCR *dcr) = 0; + virtual bool end_of_job(DCR *dcr) = 0; + + virtual bool get_cloud_volume_parts_list(DCR *dcr, const char* VolumeName, ilist *parts, POOLMEM *&err) = 0; + virtual bool get_cloud_volumes_list(DCR* dcr, alist *volumes, POOLMEM *&err) = 0; /* TODO: Adapt the prototype to have a handler instead */ + + uint32_t max_upload_retries; +}; + +#endif /* _CLOUD_DRIVER_H_ */ diff --git a/bacula/src/stored/cloud_parts.c b/bacula/src/stored/cloud_parts.c new file mode 100644 index 0000000000..259218dd81 --- /dev/null +++ b/bacula/src/stored/cloud_parts.c @@ -0,0 +1,18 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ diff --git a/bacula/src/stored/cloud_parts.h b/bacula/src/stored/cloud_parts.h new file mode 100644 index 0000000000..d70e1064c9 --- /dev/null +++ b/bacula/src/stored/cloud_parts.h @@ -0,0 +1,23 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Routines for managing Volumes contains and comparing parts + * + * Written by Norbert Bizet, May MMXVI + */ diff --git a/bacula/src/stored/cloud_test.c b/bacula/src/stored/cloud_test.c new file mode 100644 index 0000000000..8ebb71ac2a --- /dev/null +++ b/bacula/src/stored/cloud_test.c @@ -0,0 +1,19 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + diff --git a/bacula/src/stored/cloud_transfer_mgr.c b/bacula/src/stored/cloud_transfer_mgr.c new file mode 100644 index 0000000000..21357056fa --- /dev/null +++ b/bacula/src/stored/cloud_transfer_mgr.c @@ -0,0 +1,26 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula Cloud Transfer Manager: + * transfer manager wraps around the work queue. + * Reports transfer status and error + * Reports statistics about current past and future work + * Written by Norbert Bizet, May MMXVI + * + */ diff --git a/bacula/src/stored/cloud_transfer_mgr.h b/bacula/src/stored/cloud_transfer_mgr.h new file mode 100644 index 0000000000..21357056fa --- /dev/null +++ b/bacula/src/stored/cloud_transfer_mgr.h @@ -0,0 +1,26 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Bacula Cloud Transfer Manager: + * transfer manager wraps around the work queue. + * Reports transfer status and error + * Reports statistics about current past and future work + * Written by Norbert Bizet, May MMXVI + * + */ diff --git a/bacula/src/stored/dev.c b/bacula/src/stored/dev.c index 7b54e03226..187a083af1 100644 --- a/bacula/src/stored/dev.c +++ b/bacula/src/stored/dev.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -37,7 +37,6 @@ * The purpose of this code is to develop a SIMPLE Storage * daemon. More complicated coding (double buffering, writer * thread, ...) is left for a later version. - * */ /* @@ -75,6 +74,8 @@ #define O_NONBLOCK 0 #endif +static const int dbglvl = 150; + /* Imported functions */ extern void set_os_device_parameters(DCR *dcr); extern bool dev_get_os_pos(DEVICE *dev, struct mtget *mt_stat); @@ -82,291 +83,36 @@ extern uint32_t status_dev(DEVICE *dev); /* Forward referenced functions */ const char *mode_to_str(int mode); -DEVICE *m_init_dev(JCR *jcr, DEVRES *device); -/* - * Device types for printing - */ -static const char *prt_dev_types[] = { - "*none*", - "file", - "tape", - "DVD", - "FIFO", - "Vtape", - "FTP", - "VTL" -}; +DEVICE *m_init_dev(JCR *jcr, DEVRES *device, bool adata); /* - * Allocate and initialize the DEVICE structure - * Note, if dev is non-NULL, it is already allocated, - * thus we neither allocate it nor free it. This allows - * the caller to put the packet in shared memory. - * - * Note, for a tape, the device->device_name is the device name - * (e.g. /dev/nst0), and for a file, the device name - * is the directory in which the file will be placed. - * + * Device specific initialization. */ -DEVICE *init_dev(JCR *jcr, DEVRES *device) -{ - generate_global_plugin_event(bsdGlobalEventDeviceInit, device); - DEVICE *dev = m_init_dev(jcr, device); - return dev; -} - -DEVICE *m_init_dev(JCR *jcr, DEVRES *device) +void DEVICE::device_specific_init(JCR *jcr, DEVRES *device) { - struct stat statp; - int errstat; - DCR *dcr = NULL; - DEVICE *dev = NULL; - uint32_t max_bs; - - /* If no device type specified, try to guess */ - if (!device->dev_type) { - /* Check that device is available */ - if (stat(device->device_name, &statp) < 0) { - berrno be; - Jmsg2(jcr, M_ERROR, 0, _("Unable to stat device %s: ERR=%s\n"), - device->device_name, be.bstrerror()); - return NULL; - } - if (S_ISDIR(statp.st_mode)) { - device->dev_type = B_FILE_DEV; - } else if (S_ISCHR(statp.st_mode)) { - device->dev_type = B_TAPE_DEV; - } else if (S_ISFIFO(statp.st_mode)) { - device->dev_type = B_FIFO_DEV; -#ifdef USE_VTAPE - /* must set DeviceType = Vtape - * in normal mode, autodetection is disabled - */ - } else if (S_ISREG(statp.st_mode)) { - device->dev_type = B_VTAPE_DEV; -#endif - } else if (!(device->cap_bits & CAP_REQMOUNT)) { - Jmsg2(jcr, M_ERROR, 0, _("%s is an unknown device type. Must be tape or directory\n" - " or have RequiresMount=yes for DVD. st_mode=%x\n"), - device->device_name, statp.st_mode); - return NULL; - } else { - device->dev_type = B_DVD_DEV; - } - if (strcmp(device->device_name, "/dev/null") == 0) { - device->dev_type = B_NULL_DEV; - } - } - switch (device->dev_type) { - case B_DVD_DEV: - Jmsg0(jcr, M_FATAL, 0, _("DVD support is now deprecated.\n")); - return NULL; - case B_VTAPE_DEV: - dev = New(vtape); - break; -#ifdef USE_FTP - case B_FTP_DEV: - dev = New(ftp_device); - break; -#endif - case B_TAPE_DEV: - dev = New(tape_dev); - break; - case B_FILE_DEV: - case B_FIFO_DEV: - case B_NULL_DEV: - dev = New(file_dev); - break; - default: - return NULL; - } - dev->clear_slot(); /* unknown */ - - /* Copy user supplied device parameters from Resource */ - dev->dev_name = get_memory(strlen(device->device_name)+1); - pm_strcpy(dev->dev_name, device->device_name); - dev->prt_name = get_memory(strlen(device->device_name) + strlen(device->hdr.name) + 20); - /* We edit "Resource-name" (physical-name) */ - Mmsg(dev->prt_name, "\"%s\" (%s)", device->hdr.name, device->device_name); - Dmsg1(400, "Allocate dev=%s\n", dev->print_name()); - dev->capabilities = device->cap_bits; - dev->min_free_space = device->min_free_space; - dev->min_block_size = device->min_block_size; - dev->max_block_size = device->max_block_size; - dev->max_volume_size = device->max_volume_size; - dev->max_file_size = device->max_file_size; - dev->max_concurrent_jobs = device->max_concurrent_jobs; - dev->volume_capacity = device->volume_capacity; - dev->max_rewind_wait = device->max_rewind_wait; - dev->max_open_wait = device->max_open_wait; - dev->vol_poll_interval = device->vol_poll_interval; - dev->max_spool_size = device->max_spool_size; - dev->drive_index = device->drive_index; - dev->enabled = device->enabled; - dev->autoselect = device->autoselect; - dev->read_only = device->read_only; - dev->dev_type = device->dev_type; - dev->device = device; - if (dev->is_tape()) { /* No parts on tapes */ - dev->max_part_size = 0; - } else { - dev->max_part_size = device->max_part_size; - } - /* Sanity check */ - if (dev->vol_poll_interval && dev->vol_poll_interval < 60) { - dev->vol_poll_interval = 60; - } - - if (!device->dev) { - /* The first time we create a DEVICE from the DEVRES, we keep a pointer - * to the DEVICE accessible from the DEVRES. - */ - device->dev = dev; - } - - if (dev->is_fifo()) { - dev->capabilities |= CAP_STREAM; /* set stream device */ - } - - /* If the device requires mount : - * - Check that the mount point is available - * - Check that (un)mount commands are defined - */ - if (dev->is_file() && dev->requires_mount()) { - if (!device->mount_point || stat(device->mount_point, &statp) < 0) { - berrno be; - dev->dev_errno = errno; - Jmsg2(jcr, M_ERROR_TERM, 0, _("Unable to stat mount point %s: ERR=%s\n"), - device->mount_point, be.bstrerror()); - } - - if (!device->mount_command || !device->unmount_command) { - Jmsg0(jcr, M_ERROR_TERM, 0, _("Mount and unmount commands must defined for a device which requires mount.\n")); - } - } - - /* Keep the device ID in the DEVICE struct to identify the hardware */ - if (dev->is_file() && stat(dev->archive_name(), &statp) == 0) { - dev->devno = statp.st_dev; - } - - /* Sanity check */ - if (dev->max_block_size == 0) { - max_bs = DEFAULT_BLOCK_SIZE; - } else { - max_bs = dev->max_block_size; - } - if (dev->min_block_size > max_bs) { - Jmsg(jcr, M_ERROR_TERM, 0, _("Min block size > max on device %s\n"), - dev->print_name()); - } - if (dev->max_block_size > 4096000) { - Jmsg3(jcr, M_ERROR, 0, _("Block size %u on device %s is too large, using default %u\n"), - dev->max_block_size, dev->print_name(), DEFAULT_BLOCK_SIZE); - dev->max_block_size = 0; - } - if (dev->max_block_size % TAPE_BSIZE != 0) { - Jmsg3(jcr, M_WARNING, 0, _("Max block size %u not multiple of device %s block size=%d.\n"), - dev->max_block_size, dev->print_name(), TAPE_BSIZE); - } - if (dev->max_volume_size != 0 && dev->max_volume_size < (dev->max_block_size << 4)) { - Jmsg(jcr, M_ERROR_TERM, 0, _("Max Vol Size < 8 * Max Block Size for device %s\n"), - dev->print_name()); - } - - dev->errmsg = get_pool_memory(PM_EMSG); - *dev->errmsg = 0; - - if ((errstat = dev->init_mutex()) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = pthread_cond_init(&dev->wait, NULL)) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init cond variable: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = pthread_cond_init(&dev->wait_next_vol, NULL)) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init cond variable: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = pthread_mutex_init(&dev->spool_mutex, NULL)) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init spool mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = dev->init_acquire_mutex()) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init acquire mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = dev->init_freespace_mutex()) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init freespace mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = dev->init_read_acquire_mutex()) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init read acquire mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = dev->init_volcat_mutex()) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init volcat mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - if ((errstat = dev->init_dcrs_mutex()) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init dcrs mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } - - dev->set_mutex_priorities(); - -#ifdef xxx - if ((errstat = rwl_init(&dev->lock)) != 0) { - berrno be; - dev->dev_errno = errstat; - Mmsg1(dev->errmsg, _("Unable to init mutex: ERR=%s\n"), be.bstrerror(errstat)); - Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); - } -#endif - - dev->clear_opened(); - dev->attached_dcrs = New(dlist(dcr, &dcr->dev_link)); - Dmsg2(100, "init_dev: tape=%d dev_name=%s\n", dev->is_tape(), dev->dev_name); - dev->initiated = true; - - return dev; } /* - * Open the device with the operating system and + * Initialize the device with the operating system and * initialize buffer pointers. * - * Returns: true on success - * false on error + * Returns: true device already open + * false device setup but not open * * Note, for a tape, the VolName is the name we give to the * volume (not really used here), but for a file, the * VolName represents the name of the file to be created/opened. * In the case of a file, the full name is the device name * (archive_name) with the VolName concatenated. + * + * This is generic common code. It should be called prior to any + * device specific code. Note! This does not open anything. */ -bool DEVICE::open(DCR *dcr, int omode) +bool DEVICE::open_device(DCR *dcr, int omode) { - int preserve = 0; + Enter(dbglvl); + preserve = 0; + ASSERT2(!adata, "Attempt to open adata dev"); if (is_open()) { if (openmode == omode) { return true; @@ -377,6 +123,7 @@ bool DEVICE::open(DCR *dcr, int omode) preserve = state & (ST_LABEL|ST_APPEND|ST_READ); } } + openmode = omode; if (dcr) { dcr->setVolCatName(dcr->VolumeName); VolCatInfo = dcr->VolCatInfo; /* structure assign */ @@ -385,20 +132,10 @@ bool DEVICE::open(DCR *dcr, int omode) state &= ~(ST_NOSPACE|ST_LABEL|ST_APPEND|ST_READ|ST_EOT|ST_WEOT|ST_EOF); label_type = B_BACULA_LABEL; - if (is_tape() || is_fifo()) { - open_tape_device(dcr, omode); - } else if (is_ftp()) { - open_device(dcr, omode); - } else { - Dmsg1(100, "call open_file_device mode=%s\n", mode_to_str(omode)); - open_file_device(dcr, omode); + if (openmode == OPEN_READ_WRITE && has_cap(CAP_STREAM)) { + openmode = OPEN_WRITE_ONLY; } - state |= preserve; /* reset any important state info */ - Dmsg2(100, "preserve=0x%x fd=%d\n", preserve, m_fd); - - Dmsg7(100, "open dev: fd=%d dev=%p dcr=%p vol=%s type=%d dev_name=%s mode=%s\n", - m_fd, getVolCatName(), this, dcr, dev_type, print_name(), mode_to_str(omode)); - return m_fd >= 0; + return false; } void DEVICE::set_mode(int new_mode) @@ -417,17 +154,10 @@ void DEVICE::set_mode(int new_mode) mode = O_WRONLY | O_BINARY; break; default: - Emsg0(M_ABORT, 0, _("Illegal mode given to open dev.\n")); + Jmsg0(NULL, M_ABORT, 0, _("Illegal mode given to open dev.\n")); } } - -void DEVICE::open_device(DCR *dcr, int omode) -{ - /* do nothing waiting to split open_file/tape_device */ -} - - /* * Called to indicate that we have just read an * EOF from the device. @@ -435,9 +165,6 @@ void DEVICE::open_device(DCR *dcr, int omode) void DEVICE::set_ateof() { set_eof(); - if (is_tape()) { - file++; - } file_addr = 0; file_size = 0; block_num = 0; @@ -457,7 +184,7 @@ void DEVICE::set_ateot() /* - * Set the position of the device -- only for files and DVD + * Set the position of the device -- only for files * For other devices, there is no generic way to do it. * Returns: true on succes * false on error @@ -506,11 +233,6 @@ void DEVICE::clear_slot() if (vol) vol->set_slot(-1); } -const char *DEVICE::print_type() const -{ - return prt_dev_types[device->dev_type]; -} - /* * Set to unload the current volume in the drive */ @@ -518,7 +240,6 @@ void DEVICE::set_unload() { if (!m_unload && VolHdr.VolumeName[0] != 0) { m_unload = true; - memcpy(UnloadVolName, VolHdr.VolumeName, sizeof(UnloadVolName)); notify_newvol_in_attached_dcrs(NULL); } } @@ -533,210 +254,22 @@ void DEVICE::clear_volhdr() setVolCatInfo(false); } -void DEVICE::set_nospace() -{ - state |= ST_NOSPACE; -} - -void DEVICE::clear_nospace() -{ - state &= ~ST_NOSPACE; -} - -/* Put device in append mode */ -void DEVICE::set_append() -{ - state &= ~(ST_NOSPACE|ST_READ|ST_EOT|ST_EOF|ST_WEOT); /* remove EOF/EOT flags */ - state |= ST_APPEND; -} - -/* Clear append mode */ -void DEVICE::clear_append() -{ - state &= ~ST_APPEND; -} - -/* Put device in read mode */ -void DEVICE::set_read() -{ - state &= ~(ST_APPEND|ST_EOT|ST_EOF|ST_WEOT); /* remove EOF/EOT flags */ - state |= ST_READ; -} - -/* Clear read mode */ -void DEVICE::clear_read() -{ - state &= ~ST_READ; -} - -/* - * Get freespace using OS calls - * TODO: See if it's working with mount commands - */ -bool DEVICE::get_os_device_freespace() -{ - int64_t freespace, totalspace; - - if (!is_file()) { - return true; - } - if (fs_get_free_space(dev_name, &freespace, &totalspace) == 0) { - set_freespace(freespace, totalspace, 0, true); - Mmsg(errmsg, ""); - return true; - - } else { - set_freespace(0, 0, 0, false); /* No valid freespace */ - } - return false; -} - -/* Update the free space on the device */ -bool DEVICE::update_freespace() -{ - POOL_MEM ocmd(PM_FNAME); - POOLMEM* results; - char* icmd; - char* p; - uint64_t free, total; - char ed1[50]; - bool ok = false; - int status; - berrno be; - - if (!is_file()) { - Mmsg(errmsg, ""); - return true; - } - - /* The device must be mounted in order for freespace to work */ - if (requires_mount()) { - mount(1); - } - - if (get_os_device_freespace()) { - Dmsg4(20, "get_os_device_freespace: free_space=%s freespace_ok=%d free_space_errno=%d have_media=%d\n", - edit_uint64(free_space, ed1), !!is_freespace_ok(), free_space_errno, !!have_media()); - return true; - } - - icmd = device->free_space_command; - - if (!icmd) { - set_freespace(0, 0, 0, false); - Dmsg2(20, "ERROR: update_free_space_dev: free_space=%s, free_space_errno=%d (!icmd)\n", - edit_uint64(free_space, ed1), free_space_errno); - Mmsg(errmsg, _("No FreeSpace command defined.\n")); - return false; - } - - edit_mount_codes(ocmd, icmd); - - Dmsg1(20, "update_freespace: cmd=%s\n", ocmd.c_str()); - - results = get_pool_memory(PM_MESSAGE); - - Dmsg1(20, "Run freespace prog=%s\n", ocmd.c_str()); - status = run_program_full_output(ocmd.c_str(), max_open_wait/2, results); - Dmsg2(20, "Freespace status=%d result=%s\n", status, results); - /* Should report "1223232 12323232\n" "free total\n" */ - if (status == 0) { - free = str_to_int64(results) * 1024; - p = results; - - if (skip_nonspaces(&p)) { - total = str_to_int64(p) * 1024; - - } else { - total = 0; - } - - Dmsg1(400, "Free space program run: Freespace=%s\n", results); - if (free >= 0) { - set_freespace(free, total, 0, true); /* have valid freespace */ - Mmsg(errmsg, ""); - ok = true; - } - } else { - set_freespace(0, 0, EPIPE, false); /* no valid freespace */ - Mmsg2(errmsg, _("Cannot run free space command. Results=%s ERR=%s\n"), - results, be.bstrerror(status)); - - dev_errno = free_space_errno; - Dmsg4(20, "Cannot get free space on device %s. free_space=%s, " - "free_space_errno=%d ERR=%s\n", - print_name(), edit_uint64(free_space, ed1), - free_space_errno, errmsg); - } - free_pool_memory(results); - Dmsg4(20, "leave update_freespace: free_space=%s freespace_ok=%d free_space_errno=%d have_media=%d\n", - edit_uint64(free_space, ed1), !!is_freespace_ok(), free_space_errno, !!have_media()); - return ok; -} - -void DEVICE::updateVolCatBytes(uint64_t bytes) -{ - DEVICE *dev = this; - Lock_VolCatInfo(); - dev->VolCatInfo.VolCatAmetaBytes += bytes; - Dmsg1(200, "updateVolBytes ameta=%lld\n", - dev->VolCatInfo.VolCatAmetaBytes); - dev->VolCatInfo.VolCatBytes += bytes; - setVolCatInfo(false); - Unlock_VolCatInfo(); -} - -void DEVICE::updateVolCatBlocks(uint32_t blocks) -{ - DEVICE *dev = this; - Lock_VolCatInfo(); - dev->VolCatInfo.VolCatAmetaBlocks += blocks; - dev->VolCatInfo.VolCatBlocks += blocks; - setVolCatInfo(false); - Unlock_VolCatInfo(); -} - - -void DEVICE::updateVolCatWrites(uint32_t writes) -{ - DEVICE *dev = this; - Lock_VolCatInfo(); - dev->VolCatInfo.VolCatAmetaWrites += writes; - dev->VolCatInfo.VolCatWrites += writes; - setVolCatInfo(false); - Unlock_VolCatInfo(); -} - -void DEVICE::updateVolCatReads(uint32_t reads) -{ - DEVICE *dev = this; - Lock_VolCatInfo(); - dev->VolCatInfo.VolCatAmetaReads += reads; - dev->VolCatInfo.VolCatReads += reads; - setVolCatInfo(false); - Unlock_VolCatInfo(); -} - -void DEVICE::updateVolCatReadBytes(uint64_t bytes) +void DEVICE::set_volcatinfo_from_dcr(DCR *dcr) { - DEVICE *dev = this; - Lock_VolCatInfo(); - dev->VolCatInfo.VolCatAmetaRBytes += bytes; - dev->VolCatInfo.VolCatRBytes += bytes; - setVolCatInfo(false); - Unlock_VolCatInfo(); + VolCatInfo = dcr->VolCatInfo; } /* * Close the device + * Can enter with dcr==NULL */ -bool DEVICE::close() +bool DEVICE::close(DCR *dcr) { bool ok = true; - Dmsg4(40, "close_dev vol=%s fd=%d dev=%p dev=%s\n", - VolHdr.VolumeName, m_fd, this, print_name()); - offline_or_rewind(); + Dmsg5(40, "close_dev vol=%s fd=%d dev=%p adata=%d dev=%s\n", + VolHdr.VolumeName, m_fd, this, adata, print_name()); + offline_or_rewind(dcr); if (!is_open()) { Dmsg2(200, "device %s already closed vol=%s\n", print_name(), @@ -758,18 +291,14 @@ bool DEVICE::close() print_name(), be.bstrerror()); ok = false; } - break; + break; } unmount(1); /* do unmount if required */ - + /* Clean up device packet so it can be reused */ clear_opened(); - /* - * Be careful not to clear items needed by the DVD driver - * when it is closing a single part. - */ state &= ~(ST_LABEL|ST_READ|ST_APPEND|ST_EOT|ST_WEOT|ST_EOF| ST_NOSPACE|ST_MOUNTED|ST_MEDIA|ST_SHORT); label_type = B_BACULA_LABEL; @@ -788,51 +317,29 @@ bool DEVICE::close() } /* - * This call closes the device, but it is used in DVD handling - * where we close one part and then open the next part. The - * difference between close_part() and close() is that close_part() - * saves the state information of the device (e.g. the Volume lable, - * the Volume Catalog record, ... This permits opening and closing - * the Volume parts multiple times without losing track of what the - * main Volume parameters are. - */ -void DEVICE::close_part(DCR * /*dcr*/) -{ - VOLUME_LABEL saveVolHdr; - VOLUME_CAT_INFO saveVolCatInfo; /* Volume Catalog Information */ - - - saveVolHdr = VolHdr; /* structure assignment */ - saveVolCatInfo = VolCatInfo; /* structure assignment */ - close(); /* close current part */ - VolHdr = saveVolHdr; /* structure assignment */ - VolCatInfo = saveVolCatInfo; /* structure assignment */ -} - -/* * If timeout, wait until the mount command returns 0. * If !timeout, try to mount the device only once. */ bool DEVICE::mount(int timeout) { - Dmsg0(190, "Enter mount\n"); + Enter(dbglvl); if (!is_mounted() && device->mount_command) { return mount_file(1, timeout); - } + } return true; } -/* - * Unmount the device +/* + * Unmount the device * If timeout, wait until the unmount command returns 0. * If !timeout, try to unmount the device only once. */ bool DEVICE::unmount(int timeout) { - Dmsg0(100, "Enter unmount\n"); + Enter(dbglvl); if (is_mounted() && requires_mount() && device->unmount_command) { return mount_file(0, timeout); - } + } return true; } @@ -870,6 +377,9 @@ void DEVICE::edit_mount_codes(POOL_MEM &omsg, const char *imsg) str = dev_name; break; case 'e': + str = "0"; + /* ***FIXME*** this may be useful for Cloud */ +#ifdef xxx if (num_dvd_parts == 0) { if (truncating || blank_dvd) { str = "2"; @@ -879,6 +389,7 @@ void DEVICE::edit_mount_codes(POOL_MEM &omsg, const char *imsg) } else { str = "0"; } +#endif break; case 'n': bsnprintf(add, sizeof(add), "%d", part); @@ -940,7 +451,7 @@ ssize_t DEVICE::read(void *buf, size_t len) /* write to fd */ ssize_t DEVICE::write(const void *buf, size_t len) { - ssize_t write_len ; + ssize_t write_len; get_timer_count(); @@ -966,7 +477,7 @@ const char *DEVICE::name() const uint32_t DEVICE::get_file() { - if (is_dvd() || is_tape()) { + if (is_tape()) { return file; } else { uint64_t bytes = VolCatInfo.VolCatAdataBytes + VolCatInfo.VolCatAmetaBytes; @@ -976,7 +487,7 @@ uint32_t DEVICE::get_file() uint32_t DEVICE::get_block_num() { - if (is_dvd() || is_tape()) { + if (is_tape()) { return block_num; } else { return VolCatInfo.VolCatAdataBlocks + VolCatInfo.VolCatAmetaBlocks; @@ -992,7 +503,7 @@ void DEVICE::notify_newvol_in_attached_dcrs(const char *newVolumeName) { Dmsg2(140, "Notify dcrs of vol change. oldVolume=%s NewVolume=%s\n", - getVolCatName(), newVolumeName?"*None*":newVolumeName); + getVolCatName(), newVolumeName?newVolumeName:"*None*"); Lock_dcrs(); DCR *mdcr; foreach_dlist(mdcr, attached_dcrs) { @@ -1032,16 +543,24 @@ DEVICE::notify_newfile_in_attached_dcrs() /* * Free memory allocated for the device + * Can enter with dcr==NULL */ -void DEVICE::term(void) +void DEVICE::term(DCR *dcr) { - DEVICE *dev = NULL; Dmsg1(900, "term dev: %s\n", print_name()); - close(); + if (!dcr) { + d_close(m_fd); + } else { + close(dcr); + } if (dev_name) { free_memory(dev_name); dev_name = NULL; } + if (adev_name) { + free_memory(adev_name); + adev_name = NULL; + } if (prt_name) { free_memory(prt_name); prt_name = NULL; @@ -1064,9 +583,6 @@ void DEVICE::term(void) device->dev = NULL; } delete this; - if (dev) { - dev->term(); - } } /* Get freespace values */ @@ -1098,78 +614,437 @@ void DEVICE::set_freespace(uint64_t freeval, uint64_t totalval, int errnoval, bo V(freespace_mutex); } -/* - * This routine initializes the device wait timers +/* Convenient function that return true only if the device back-end is a + * filesystem that its nearly full. (the free space is below the given threshold) */ -void init_device_wait_timers(DCR *dcr) +bool DEVICE::is_fs_nearly_full(uint64_t threshold) { - DEVICE *dev = dcr->dev; - JCR *jcr = dcr->jcr; + uint64_t freeval, totalval; + if (is_file()) { + get_freespace(&freeval, &totalval); + if (totalval > 0) { + if (freeval < threshold) { + return true; + } + } + } + return false; +} - /* ******FIXME******* put these on config variables */ - dev->min_wait = 60 * 60; - dev->max_wait = 24 * 60 * 60; - dev->max_num_wait = 9; /* 5 waits =~ 1 day, then 1 day at a time */ - dev->wait_sec = dev->min_wait; - dev->rem_wait_sec = dev->wait_sec; - dev->num_wait = 0; - dev->poll = false; +static const char *modes[] = { + "CREATE_READ_WRITE", + "OPEN_READ_WRITE", + "OPEN_READ_ONLY", + "OPEN_WRITE_ONLY" +}; - jcr->min_wait = 60 * 60; - jcr->max_wait = 24 * 60 * 60; - jcr->max_num_wait = 9; /* 5 waits =~ 1 day, then 1 day at a time */ - jcr->wait_sec = jcr->min_wait; - jcr->rem_wait_sec = jcr->wait_sec; - jcr->num_wait = 0; +const char *mode_to_str(int mode) +{ + static char buf[100]; + if (mode < 1 || mode > 4) { + bsnprintf(buf, sizeof(buf), "BAD mode=%d", mode); + return buf; + } + return modes[mode-1]; } -void init_jcr_device_wait_timers(JCR *jcr) +void DEVICE::setVolCatName(const char *name) { - /* ******FIXME******* put these on config variables */ - jcr->min_wait = 60 * 60; - jcr->max_wait = 24 * 60 * 60; - jcr->max_num_wait = 9; /* 5 waits =~ 1 day, then 1 day at a time */ - jcr->wait_sec = jcr->min_wait; - jcr->rem_wait_sec = jcr->wait_sec; - jcr->num_wait = 0; + bstrncpy(VolCatInfo.VolCatName, name, sizeof(VolCatInfo.VolCatName)); + setVolCatInfo(false); } +void DEVICE::setVolCatStatus(const char *status) +{ + bstrncpy(VolCatInfo.VolCatStatus, status, sizeof(VolCatInfo.VolCatStatus)); + setVolCatInfo(false); +} + +void DEVICE::updateVolCatBytes(uint64_t bytes) +{ + Lock_VolCatInfo(); + VolCatInfo.VolCatAmetaBytes += bytes; + VolCatInfo.VolCatBytes += bytes; + setVolCatInfo(false); + Unlock_VolCatInfo(); +} + +void DEVICE::updateVolCatHoleBytes(uint64_t hole) +{ + return; +} + +void DEVICE::updateVolCatPadding(uint64_t padding) +{ + Lock_VolCatInfo(); + VolCatInfo.VolCatAmetaPadding += padding; + VolCatInfo.VolCatPadding += padding; + setVolCatInfo(false); + Unlock_VolCatInfo(); +} + + +void DEVICE::updateVolCatBlocks(uint32_t blocks) +{ + Lock_VolCatInfo(); + VolCatInfo.VolCatAmetaBlocks += blocks; + VolCatInfo.VolCatBlocks += blocks; + setVolCatInfo(false); + Unlock_VolCatInfo(); +} + +void DEVICE::updateVolCatWrites(uint32_t writes) +{ + Lock_VolCatInfo(); + VolCatInfo.VolCatAmetaWrites += writes; + VolCatInfo.VolCatWrites += writes; + setVolCatInfo(false); + Unlock_VolCatInfo(); +} + +void DEVICE::updateVolCatReads(uint32_t reads) +{ + Lock_VolCatInfo(); + VolCatInfo.VolCatAmetaReads += reads; + VolCatInfo.VolCatReads += reads; + setVolCatInfo(false); + Unlock_VolCatInfo(); +} + +void DEVICE::updateVolCatReadBytes(uint64_t bytes) +{ + Lock_VolCatInfo(); + VolCatInfo.VolCatAmetaRBytes += bytes; + VolCatInfo.VolCatRBytes += bytes; + setVolCatInfo(false); + Unlock_VolCatInfo(); +} + +void DEVICE::set_nospace() +{ + state |= ST_NOSPACE; +} + +void DEVICE::clear_nospace() +{ + state &= ~ST_NOSPACE; +} + +/* Put device in append mode */ +void DEVICE::set_append() +{ + state &= ~(ST_NOSPACE|ST_READ|ST_EOT|ST_EOF|ST_WEOT); /* remove EOF/EOT flags */ + state |= ST_APPEND; +} + +/* Clear append mode */ +void DEVICE::clear_append() +{ + state &= ~ST_APPEND; +} + +/* Put device in read mode */ +void DEVICE::set_read() +{ + state &= ~(ST_APPEND|ST_EOT|ST_EOF|ST_WEOT); /* remove EOF/EOT flags */ + state |= ST_READ; +} + +/* Clear read mode */ +void DEVICE::clear_read() +{ + state &= ~ST_READ; +} /* - * The dev timers are used for waiting on a particular device - * - * Returns: true if time doubled - * false if max time expired + * Get freespace using OS calls + * TODO: See if it's working with mount commands */ -bool double_dev_wait_time(DEVICE *dev) +bool DEVICE::get_os_device_freespace() +{ + int64_t freespace, totalspace; + + if (!is_file()) { + return true; + } + if (fs_get_free_space(dev_name, &freespace, &totalspace) == 0) { + set_freespace(freespace, totalspace, 0, true); + Mmsg(errmsg, ""); + return true; + + } else { + set_freespace(0, 0, 0, false); /* No valid freespace */ + } + return false; +} + +/* Update the free space on the device */ +bool DEVICE::update_freespace() { - dev->wait_sec *= 2; /* double wait time */ - if (dev->wait_sec > dev->max_wait) { /* but not longer than maxtime */ - dev->wait_sec = dev->max_wait; + POOL_MEM ocmd(PM_FNAME); + POOLMEM* results; + char* icmd; + char* p; + uint64_t free, total; + char ed1[50]; + bool ok = false; + int status; + berrno be; + + if (!is_file()) { + Mmsg(errmsg, ""); + return true; + } + + /* The device must be mounted in order for freespace to work */ + if (requires_mount()) { + mount(1); } - dev->num_wait++; - dev->rem_wait_sec = dev->wait_sec; - if (dev->num_wait >= dev->max_num_wait) { + + if (get_os_device_freespace()) { + Dmsg4(20, "get_os_device_freespace: free_space=%s freespace_ok=%d free_space_errno=%d have_media=%d\n", + edit_uint64(free_space, ed1), !!is_freespace_ok(), free_space_errno, !!have_media()); + return true; + } + + icmd = device->free_space_command; + + if (!icmd) { + set_freespace(0, 0, 0, false); + Dmsg2(20, "ERROR: update_free_space_dev: free_space=%s, free_space_errno=%d (!icmd)\n", + edit_uint64(free_space, ed1), free_space_errno); + Mmsg(errmsg, _("No FreeSpace command defined.\n")); + return false; + } + + edit_mount_codes(ocmd, icmd); + + Dmsg1(20, "update_freespace: cmd=%s\n", ocmd.c_str()); + + results = get_pool_memory(PM_MESSAGE); + + Dmsg1(20, "Run freespace prog=%s\n", ocmd.c_str()); + status = run_program_full_output(ocmd.c_str(), max_open_wait/2, results); + Dmsg2(20, "Freespace status=%d result=%s\n", status, results); + /* Should report "1223232 12323232\n" "free total\n" */ + if (status == 0) { + free = str_to_int64(results) * 1024; + p = results; + + if (skip_nonspaces(&p)) { + total = str_to_int64(p) * 1024; + + } else { + total = 0; + } + + Dmsg1(400, "Free space program run: Freespace=%s\n", results); + if (free >= 0) { + set_freespace(free, total, 0, true); /* have valid freespace */ + Mmsg(errmsg, ""); + ok = true; + } + } else { + set_freespace(0, 0, EPIPE, false); /* no valid freespace */ + Mmsg2(errmsg, _("Cannot run free space command. Results=%s ERR=%s\n"), + results, be.bstrerror(status)); + + dev_errno = free_space_errno; + Dmsg4(20, "Cannot get free space on device %s. free_space=%s, " + "free_space_errno=%d ERR=%s\n", + print_name(), edit_uint64(free_space, ed1), + free_space_errno, errmsg); + } + free_pool_memory(results); + Dmsg4(20, "leave update_freespace: free_space=%s freespace_ok=%d free_space_errno=%d have_media=%d\n", + edit_uint64(free_space, ed1), !!is_freespace_ok(), free_space_errno, !!have_media()); + return ok; +} + +bool DEVICE::weof(DCR */*dcr*/, int num) +{ + Dmsg1(129, "=== weof_dev=%s\n", print_name()); + + if (!is_open()) { + dev_errno = EBADF; + Mmsg0(errmsg, _("Bad call to weof_dev. Device not open\n")); + Emsg0(M_FATAL, 0, errmsg); + return false; + } + + if (!can_append()) { + Mmsg0(errmsg, _("Attempt to WEOF on non-appendable Volume\n")); + Emsg0(M_FATAL, 0, errmsg); return false; } + + file_size = 0; return true; } -static const char *modes[] = { - "CREATE_READ_WRITE", - "OPEN_READ_WRITE", - "OPEN_READ_ONLY", - "OPEN_WRITE_ONLY" -}; +/* + * Very basic functions -- no device specific code. + * Returns: true on succes + * false on error + */ +bool DEVICE::eod(DCR *dcr) +{ + bool ok = true; + + Enter(dbglvl); + if (m_fd < 0) { + dev_errno = EBADF; + Mmsg1(errmsg, _("Bad call to eod. Device %s not open\n"), print_name()); + Dmsg1(100, "%s", errmsg); + return false; + } + if (at_eot()) { + Leave(100); + return true; + } + clear_eof(); /* remove EOF flag */ + block_num = file = 0; + file_size = 0; + file_addr = 0; + Leave(100); + return ok; +} -const char *mode_to_str(int mode) +bool DEVICE::is_eod_valid(DCR *dcr) { - static char buf[100]; - if (mode < 1 || mode > 4) { - bsnprintf(buf, sizeof(buf), "BAD mode=%d", mode); - return buf; - } - return modes[mode-1]; + return true; +} + +bool DEVICE::open_next_part(DCR */*dcr*/) +{ + return true; +} + +bool DEVICE::close_part(DCR */*dcr*/) +{ + return true; +} + +DEVICE *DEVICE::get_dev(DCR */*dcr*/) +{ + return this; +} + +uint32_t DEVICE::get_hi_addr() +{ + return (uint32_t)(file_addr >> 32); +} + +uint32_t DEVICE::get_hi_addr(boffset_t addr) +{ + return (uint32_t)(addr >> 32); +} + +uint32_t DEVICE::get_low_addr() +{ + return (uint32_t)(file_addr); +} + +uint32_t DEVICE::get_low_addr(boffset_t addr) +{ + return (uint32_t)(addr); +} + + +uint64_t DEVICE::get_full_addr() +{ + return file_addr; +} + +uint64_t DEVICE::get_full_addr(boffset_t addr) +{ + return addr; +} + + +uint64_t DEVICE::get_full_addr(uint32_t hi, uint32_t low) +{ + return ((uint64_t)hi)<<32 | (uint64_t)low; +} + +/* Note: this subroutine is not in the class */ +uint64_t get_full_addr(uint32_t hi, uint32_t low) +{ + return ((uint64_t)hi)<<32 | (uint64_t)low; +} + +/* Print the file address */ +char *DEVICE::print_addr(char *buf, int32_t buf_len) +{ + buf[0] = 0; + bsnprintf(buf, buf_len, "%llu", get_full_addr()); + return buf; +} + +char *DEVICE::print_addr(char *buf, int32_t buf_len, boffset_t addr) +{ + buf[0] = 0; + bsnprintf(buf, buf_len, "%llu", addr); + return buf; +} + + +bool DEVICE::do_size_checks(DCR *dcr, DEV_BLOCK *block) +{ + JCR *jcr = dcr->jcr; + + if (is_user_volume_size_reached(dcr, true)) { + Dmsg0(40, "Calling terminate_writing_volume\n"); + terminate_writing_volume(dcr); + reread_last_block(dcr); /* Only used on tapes */ + dev_errno = ENOSPC; + return false; + } + + /* + * Limit maximum File size on volume to user specified value. + * In practical terms, this means to put an EOF mark on + * a tape after every X bytes. This effectively determines + * how many index records we have (JobMedia). If you set + * max_file_size too small, it will cause a lot of shoe-shine + * on very fast modern tape (LTO-3 and above). + */ + if ((max_file_size > 0) && + (file_size+block->binbuf) >= max_file_size) { + file_size = 0; /* reset file size */ + + if (!weof(dcr, 1)) { /* write eof */ + Dmsg0(50, "WEOF error in max file size.\n"); + Jmsg(jcr, M_FATAL, 0, _("Unable to write EOF. ERR=%s\n"), + bstrerror()); + Dmsg0(40, "Calling terminate_writing_volume\n"); + terminate_writing_volume(dcr); + dev_errno = ENOSPC; + return false; + } + + if (!do_new_file_bookkeeping(dcr)) { + /* Error message already sent */ + return false; + } + } + return true; +} + +bool DEVICE::get_tape_alerts(DCR *dcr) +{ + return true; +} + +void DEVICE::show_tape_alerts(DCR *dcr, alert_list_type type, + alert_list_which which, alert_cb alert_callback) +{ + return; +} + +int DEVICE::delete_alerts() +{ + return 0; } diff --git a/bacula/src/stored/dev.h b/bacula/src/stored/dev.h index 1371e385b6..127abd6b67 100644 --- a/bacula/src/stored/dev.h +++ b/bacula/src/stored/dev.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +21,6 @@ * Tape and File storage access * * Kern Sibbald, MM - * */ /* @@ -55,37 +54,40 @@ /* Return values from wait_for_sysop() */ enum { - W_ERROR = 1, - W_TIMEOUT, - W_POLL, - W_MOUNT, - W_WAKE + W_ERROR = 1, + W_TIMEOUT = 2, + W_POLL = 3, + W_MOUNT = 4, + W_WAKE = 5 }; /* Arguments to open_dev() */ enum { CREATE_READ_WRITE = 1, - OPEN_READ_WRITE, - OPEN_READ_ONLY, - OPEN_WRITE_ONLY + OPEN_READ_WRITE = 2, + OPEN_READ_ONLY = 3, + OPEN_WRITE_ONLY = 4 }; -/* - * Device types - * If you update this table, be sure to add an - * entry in prt_dev_types[] in dev.c - */ -enum { - B_FILE_DEV = 1, - B_TAPE_DEV, - B_DVD_DEV, - B_FIFO_DEV, - B_VTAPE_DEV, /* change to B_TAPE_DEV after init */ - B_FTP_DEV, - B_VTL_DEV, /* Virtual tape library device */ - B_NULL_DEV +/* Argument to print tape alerts */ +enum alert_list_type { + list_codes = 1, + list_short = 2, + list_long = 3 }; +enum alert_list_which { + list_last = 1, + list_all = 2 +}; + +typedef void (alert_cb)(void *alert_ctx, const char *short_msg, + const char *long_msg, char *Volume, int severity, + int flags, int alert, utime_t alert_time); + +/* Aligned Data Disk Volume extension */ +#define ADATA_EXTENSION ".add" + /* Generic status bits returned from status_dev() */ #define BMT_TAPE (1<<0) /* is tape device */ #define BMT_EOF (1<<1) /* just read EOF */ @@ -124,6 +126,7 @@ enum { #define CAP_REQMOUNT (1<<21) /* Require mount and unmount */ #define CAP_CHECKLABELS (1<<22) /* Check for ANSI/IBM labels */ #define CAP_BLOCKCHECKSUM (1<<23) /* Create/test block checksum */ +#define CAP_LSEEK (1<<24) /* Has lseek function defined i.e. basically File storage */ /* Test state */ #define dev_state(dev, st_state) ((dev)->state & (st_state)) @@ -175,16 +178,21 @@ struct VOLUME_CAT_INFO { uint64_t VolCatAmetaRBytes; /* Ameta bytes read */ uint64_t VolCatAdataRBytes; /* Adata bytes read */ uint64_t VolCatHoleBytes; /* Total hole bytes */ + uint64_t VolEndAddr; /* Last Volume address */ + uint64_t VolLastPartBytes; /* Bytes in last part */ uint32_t VolCatHoles; /* Number of holes */ uint32_t VolCatJobs; /* Number of jobs on this Volume */ uint32_t VolCatFiles; /* Number of files */ uint32_t VolCatType; /* Volume drive type */ + uint32_t VolCatParts; /* Max number of cache parts */ + uint32_t VolCatCloudParts; /* Max number of cloud parts */ uint32_t VolCatMounts; /* Number of mounts this volume */ uint32_t VolCatErrors; /* Number of errors this volume */ uint32_t VolCatRecycles; /* Number of recycles this volume */ uint32_t EndFile; /* Last file number */ uint32_t EndBlock; /* Last block number */ + int32_t LabelType; /* Bacula/ANSI/IBM */ int32_t Slot; /* >0=Slot loaded, 0=nothing, -1=unknown */ uint32_t VolCatMaxJobs; /* Maximum Jobs to write to volume */ @@ -199,13 +207,14 @@ struct VOLUME_CAT_INFO { utime_t VolLastWritten; /* Time of last write */ bool InChanger; /* Set if vol in current magazine */ bool is_valid; /* set if this data is valid */ + bool VolEnabled; /* set if volume enabled */ char VolCatStatus[20]; /* Volume status */ char VolCatName[MAX_NAME_LENGTH]; /* Desired volume to mount */ }; -class DEVRES; /* Device resource defined in stored_conf.h */ -class DCR; /* forward reference */ -class VOLRES; /* forward reference */ +class DEVRES; /* Device resource defined in stored_conf.h */ +class DCR; /* forward reference */ +class VOLRES; /* forward reference */ /* * Device structure definition. There is one of these for @@ -213,9 +222,7 @@ class VOLRES; /* forward reference */ * that device and effects all jobs using the device. */ class DEVICE: public SMARTALLOC { -protected: - int m_fd; /* file descriptor */ -private: +public: int m_blocked; /* set if we must wait (i.e. change tape) */ int m_count; /* Mutex use count -- DEBUG only */ int m_num_reserved; /* counter of device reservations */ @@ -228,6 +235,7 @@ private: bthread_mutex_t m_mutex; /* access control */ bthread_mutex_t acquire_mutex; /* mutex for acquire code */ pthread_mutex_t read_acquire_mutex; /* mutex for acquire read code */ + pthread_mutex_t adata_mutex; /* Lock for writing adata */ pthread_mutex_t volcat_mutex; /* VolCatInfo mutex */ pthread_mutex_t dcrs_mutex; /* Attached dcr mutex */ pthread_mutex_t freespace_mutex; /* mutex to compute the freespace */ @@ -241,6 +249,7 @@ public: pthread_cond_t wait; /* thread wait variable */ pthread_cond_t wait_next_vol; /* wait for tape to be mounted */ pthread_t no_wait_id; /* this thread must not wait */ + int m_fd; /* file descriptor */ int dev_prev_blocked; /* previous blocked state */ int num_waiting; /* number of threads waiting */ int num_writers; /* number of writing threads */ @@ -249,14 +258,23 @@ public: int dev_errno; /* Our own errno */ int mode; /* read/write modes */ int openmode; /* parameter passed to open_dev (useful to reopen the device) */ + int preserve; /* preserve open state */ int dev_type; /* device type */ + uint32_t blocked_by; /* JobId that blocked */ bool enabled; /* Set when enabled */ bool autoselect; /* Autoselect in autochanger */ bool read_only; /* Device is read only */ bool initiated; /* set when init_dev() called */ + bool m_shstore; /* Shares storage can be used */ + bool m_shstore_lock; /* set if shared lock set */ + bool m_shstore_user_lock; /* set if user set shared lock */ + bool m_shstore_register; /* set if register key set */ + bool m_shstore_blocked; /* Set if I am blocked */ + bool adata; /* set if adata device */ int label_type; /* Bacula/ANSI/IBM label types */ uint32_t drive_index; /* Autochanger drive index (base 0) */ POOLMEM *dev_name; /* Physical device name */ + POOLMEM *adev_name; /* Aligned device name */ POOLMEM *prt_name; /* Name used for display purposes */ char *errmsg; /* nicely edited error message */ uint32_t block_num; /* current block number base 0 */ @@ -266,6 +284,7 @@ public: uint64_t file_size; /* Current file size */ uint32_t EndBlock; /* last block written */ uint32_t EndFile; /* last file written */ + uint64_t EndAddr; /* Ending write addr */ uint32_t min_block_size; /* min block size */ uint32_t max_block_size; /* max block size */ uint32_t max_concurrent_jobs; /* maximum simultaneous jobs this drive */ @@ -276,21 +295,21 @@ public: uint64_t spool_size; /* current spool size for this device */ uint32_t max_rewind_wait; /* max secs to allow for rewind */ uint32_t max_open_wait; /* max secs to allow for open */ + uint32_t padding_size; /* adata block padding -- bytes */ + uint32_t file_alignment; /* adata file alignment -- bytes */ + uint32_t adata_size; /* adata block size */ + uint64_t adata_addr; /* Next adata write address */ uint64_t max_part_size; /* max part size */ uint64_t part_size; /* current part size */ uint32_t part; /* current part number (starts at 0) */ - uint64_t part_start; /* current part start address (relative to the whole volume) */ - uint32_t num_dvd_parts; /* number of parts WRITTEN on the DVD */ /* state ST_FREESPACE_OK is set if free_space is valid */ uint64_t free_space; /* current free space on device */ uint64_t total_space; /* current used space on device */ uint64_t devno; /* device id */ uint64_t min_free_space; /* Minimum free disk space */ int free_space_errno; /* indicates errno getting freespace */ - bool truncating; /* if set, we are currently truncating the DVD */ - bool blank_dvd; /* if set, we have a blank DVD in the drive */ - + bool truncating; /* if set, we are currently truncating */ utime_t vol_poll_interval; /* interval between polling Vol mount */ DEVRES *device; /* pointer to Device Resource */ @@ -298,11 +317,12 @@ public: btimer_t *tid; /* timer id */ VOLUME_CAT_INFO VolCatInfo; /* Volume Catalog Information */ - VOLUME_LABEL VolHdr; /* Actual volume label */ + VOLUME_LABEL VolHdr; /* Actual volume label (only needs to be correct when writing a new header) */ char pool_name[MAX_NAME_LENGTH]; /* pool name */ char pool_type[MAX_NAME_LENGTH]; /* pool type */ + char reserved_pool_name[MAX_NAME_LENGTH]; /* pool name for reserves */ - char UnloadVolName[MAX_NAME_LENGTH]; /* Last wrong Volume mounted */ + char LoadedVolName[MAX_NAME_LENGTH]; /* Last loaded Volume */ char lock_holder[12]; /* holder of SCSI lock */ bool poll; /* set to poll Volume */ /* Device wait times ***FIXME*** look at durations */ @@ -313,17 +333,19 @@ public: int rem_wait_sec; int num_wait; - btime_t last_timer; /* used by read/write/seek to get stats (usec) */ - btime_t last_tick; /* contains last read/write time (usec) */ + btime_t last_timer; /* used by read/write/seek to get stats (usec) */ + btime_t last_tick; /* contains last read/write time (usec) */ btime_t DevReadTime; btime_t DevWriteTime; uint64_t DevWriteBytes; uint64_t DevReadBytes; + uint64_t usage; /* Drive usage read+write bytes */ /* Methods */ - btime_t get_timer_count(); /* return the last timer interval (ms) */ + btime_t get_timer_count(); /* return the last timer interval (ms) */ + void device_generic_init(JCR *jcr, DEVRES *device); int has_cap(int cap) const { return capabilities & cap; } void clear_cap(int cap) { capabilities &= ~cap; } void set_cap(int cap) { capabilities |= cap; } @@ -333,18 +355,19 @@ public: int is_removable() const { return capabilities & CAP_REM; } bool is_tape() const { return (dev_type == B_TAPE_DEV || dev_type == B_VTAPE_DEV); } - bool is_ftp() const { return dev_type == B_FTP_DEV; } - bool is_file() const { return (dev_type == B_FILE_DEV); } + bool is_file() const { return (dev_type == B_FILE_DEV) || is_aligned() || is_cloud(); } + bool is_cloud() const { return dev_type == B_CLOUD_DEV; } + bool is_adata() const { return dev_type == B_ADATA_DEV; } + bool is_aligned() const { return dev_type == B_ALIGNED_DEV; } bool is_null() const { return dev_type == B_NULL_DEV; } bool is_fifo() const { return dev_type == B_FIFO_DEV; } - bool is_dvd() const { return dev_type == B_DVD_DEV; } bool is_vtl() const { return dev_type == B_VTL_DEV; } bool is_vtape() const { return dev_type == B_VTAPE_DEV; } bool is_open() const { return m_fd >= 0; } int is_offline() const { return state & ST_OFFLINE; } int is_labeled() const { return state & ST_LABEL; } int is_mounted() const { return state & ST_MOUNTED; } - int is_unmountable() const { return (is_dvd() || (is_file() && is_removable())); } + int is_unmountable() const { return ((is_file() && is_removable())); } int num_reserved() const { return m_num_reserved; }; int is_part_spooled() const { return state & ST_PART_SPOOLED; } int have_media() const { return state & ST_MEDIA; } @@ -378,10 +401,9 @@ public: bool must_load() const { return m_load; }; const char *strerror() const; const char *archive_name() const; + const char *aligned_name() const; const char *name() const; const char *print_name() const; /* Name for display purposes */ - const char *print_type() const; /* in dev.c */ - void set_ateof(); /* in dev.c */ void set_ateot(); /* in dev.c */ void set_eot() { state |= ST_EOT; }; void set_eof() { state |= ST_EOF; }; @@ -396,8 +418,9 @@ public: }; void get_freespace(uint64_t *freeval, uint64_t *totalval); /* in dev.c */ void set_freespace(uint64_t freeval, uint64_t totalval, int errnoval, bool valid); /* in dev.c */ + bool is_fs_nearly_full(uint64_t threshold); bool is_volume_to_unload() const { \ - return m_unload && strcmp(VolHdr.VolumeName, UnloadVolName) == 0; }; + return m_unload && strcmp(VolHdr.VolumeName, LoadedVolName) == 0; }; void set_load() { m_load = true; }; void set_wait() { m_wait = true; }; void clear_wait() { m_wait = false; }; @@ -417,21 +440,13 @@ public: void clear_media() { state &= ~ST_MEDIA; }; void clear_short_block() { state &= ~ST_SHORT; }; void clear_freespace_ok() { state &= ~ST_FREESPACE_OK; }; - void clear_unload() { m_unload = false; UnloadVolName[0] = 0; }; + void clear_unload() { m_unload = false; }; void clear_load() { m_load = false; }; char *bstrerror(void) { return errmsg; }; char *print_errmsg() { return errmsg; }; int32_t get_slot() const { return m_slot; }; void setVolCatInfo(bool valid) { VolCatInfo.is_valid = valid; }; bool haveVolCatInfo() const { return VolCatInfo.is_valid; }; - void setVolCatName(const char *name) { - bstrncpy(VolCatInfo.VolCatName, name, sizeof(VolCatInfo.VolCatName)); - setVolCatInfo(false); - }; - void setVolCatStatus(const char *status) { - bstrncpy(VolCatInfo.VolCatStatus, status, sizeof(VolCatInfo.VolCatStatus)); - setVolCatInfo(false); - }; void clearVolCatBytes() { VolCatInfo.VolCatBytes = 0; @@ -441,24 +456,14 @@ public: char *getVolCatName() { return VolCatInfo.VolCatName; }; - void set_nospace(); /* in aligned.c */ - void set_append(); /* in aligned.c */ - void set_read(); /* in aligned.c */ - void clear_nospace(); /* in aligned.c */ - void clear_append(); /* in aligned.c */ - void clear_read(); /* in aligned.c */ void set_unload(); /* in dev.c */ void clear_volhdr(); /* in dev.c */ - void close_part(DCR *dcr); /* in dev.c */ - void term(void); /* in dev.c */ ssize_t read(void *buf, size_t len); /* in dev.c */ ssize_t write(const void *buf, size_t len); /* in dev.c */ void edit_mount_codes(POOL_MEM &omsg, const char *imsg); /* in dev.c */ - bool offline_or_rewind(); /* in dev.c */ - bool eod(DCR *dcr); /* in dev.c */ + bool offline_or_rewind(DCR *dcr); /* in dev.c */ bool fsr(int num); /* in dev.c */ bool bsr(int num); /* in dev.c */ - bool weof(int num); /* in dev.c */ int32_t get_os_tape_file(); /* in dev.c */ bool scan_dir_for_volume(DCR *dcr); /* in scan.c */ void clrerror(int func); /* in dev.c */ @@ -471,21 +476,41 @@ public: void attach_dcr_to_dev(DCR *dcr); /* in acquire.c */ void detach_dcr_from_dev(DCR *dcr); /* in acquire.c */ - void updateVolCatBytes(uint64_t); /* in dev.c */ - void updateVolCatBlocks(uint32_t); /* in dev.c */ - void updateVolCatWrites(uint32_t); /* in dev.c */ - void updateVolCatReads(uint32_t); /* in dev.c */ - void updateVolCatReadBytes(uint64_t); /* in dev.c */ - void updateVolCatPadding(uint64_t); /* in dev.c */ - void updateVolCatHoleBytes(uint64_t); /* in dev.c */ uint32_t get_file(); /* in dev.c */ uint32_t get_block_num(); /* in dev.c */ int fd() const { return m_fd; }; bool mount_file(int mount, int dottimout); + void dump_volume_label(); + DEV_BLOCK *new_block(DCR *dcr, int size=0); + /* Virtual functions that can be overridden */ + virtual void setVolCatName(const char *name); + virtual void setVolCatStatus(const char *status); + virtual void free_dcr_blocks(DCR *dcr); /* in block_util.c */ + virtual void new_dcr_blocks(DCR *dcr); /* in block_util.c */ + virtual boffset_t get_adata_size(DCR *dcr) { return (boffset_t)0; }; + virtual void updateVolCatBytes(uint64_t); /* in dev.c */ + virtual void updateVolCatBlocks(uint32_t); /* in dev.c */ + virtual void updateVolCatWrites(uint32_t); /* in dev.c */ + virtual void updateVolCatReads(uint32_t); /* in dev.c */ + virtual void updateVolCatReadBytes(uint64_t); /* in dev.c */ + virtual void updateVolCatPadding(uint64_t); /* in dev.c */ + virtual void updateVolCatHoleBytes(uint64_t); /* in dev.c */ + virtual bool setVolCatAdataBytes(uint64_t) { return false; }; + virtual void set_volcatinfo_from_dcr(DCR *dcr); + virtual bool allow_maxbytes_concurrency(DCR *dcr) { return true; }; + virtual bool flush_before_eos(DCR *dcr) { return true; }; + virtual void set_nospace(); /* in dev.c */ + virtual void set_append(); /* in dev.c */ + virtual void set_read(); /* in dev.c */ + virtual void clear_nospace(); /* in dev.c */ + virtual void clear_append(); /* in dev.c */ + virtual void clear_read(); /* in dev.c */ + virtual void device_specific_init(JCR *jcr, DEVRES *device); + virtual void device_specific_open(DCR *dcr) { return; }; virtual int d_ioctl(int fd, ioctl_req_t request, char *mt_com=NULL); virtual int d_open(const char *pathname, int flags); virtual int d_close(int fd); @@ -495,53 +520,112 @@ public: virtual bool update_pos(DCR *dcr); virtual bool rewind(DCR *dcr); virtual bool truncate(DCR *dcr); - virtual void open_device(DCR *dcr, int omode); - virtual bool close(); /* in dev.c */ - virtual bool open(DCR *dcr, int mode); /* in dev.c */ + virtual int truncate_cache(DCR *dcr, const char *VolName, int64_t *size) { return 0; }; + virtual bool get_cloud_volumes_list(DCR* dcr, alist *volumes, POOLMEM *&err) { pm_strcpy(err, "Not implemented"); return false;}; + virtual bool get_cloud_volume_parts_list(DCR *dcr, const char *VolumeName, ilist *parts, POOLMEM *&err) { pm_strcpy(err, "Not implemented"); return false; }; + virtual uint32_t get_cloud_upload_transfer_status(POOL_MEM &msg, bool verbose) { pm_strcpy(msg, "Not implemented"); return 0; }; + virtual uint32_t get_cloud_download_transfer_status(POOL_MEM &msg, bool verbose) { pm_strcpy(msg, "Not implemented"); return 0; }; + virtual bool upload_cache(DCR *dcr, const char *VolName, POOLMEM *&err) { return true; }; + virtual bool open_device(DCR *dcr, int omode) = 0; + virtual bool open_next_part(DCR *dcr); + virtual bool close(DCR *dcr); /* in dev.c */ + virtual void term(DCR *dcr); /* in dev.c */ + virtual bool close_part(DCR *dcr); + virtual bool reposition(DCR *dcr, uint64_t raddr); virtual bool mount(int timeout); virtual bool unmount(int timeout); - + virtual int read_dev_volume_label(DCR *dcr); /* in label.c */ + virtual bool write_volume_label_to_block(DCR *dcr); /* in label.c */ + virtual bool write_volume_label(DCR *dcr, + const char *VolName, const char *PoolName, + bool relabel, bool no_prelabel); /* in label.c */ + virtual bool write_volume_label_to_dev(DCR *dcr, + const char *VolName, const char *PoolName, + bool relabel, bool no_prelabel); /* in label.c */ + virtual bool rewrite_volume_label(DCR *dcr, bool recycle); /* in label.c */ + virtual bool is_eod_valid(DCR *dcr); /* in dev.c */ + virtual bool eod(DCR *dcr); /* in dev.c */ + virtual bool weof(DCR *dcr, int num); /* in dev.c */ + virtual bool end_of_volume(DCR *dcr) { return true; }; + virtual bool start_of_job(DCR *dcr) {return true; }; + virtual bool end_of_job(DCR *dcr) {return true; }; + virtual bool is_indexed() { return true; }; + virtual void set_ateof(); /* in dev.c */ + virtual const char *print_type() = 0; /* in dev.c */ + virtual DEVICE *get_dev(DCR *dcr); /* in dev.c */ + virtual uint32_t get_hi_addr(); /* in dev.c */ + virtual uint32_t get_low_addr(); /* in dev.c */ + virtual uint32_t get_hi_addr(boffset_t addr); /* in dev.c */ + virtual uint32_t get_low_addr(boffset_t addr); /* in dev.c */ + virtual uint64_t get_full_addr(); /* in dev.c */ + virtual uint64_t get_full_addr(boffset_t addr); /* in dev.c */ + virtual uint64_t get_full_addr(uint32_t hi, uint32_t low); /* in dev.c */ + virtual char *print_addr(char *buf, int32_t maxlen); + virtual char *print_addr(char *buf, int32_t maxlen, boffset_t addr); + virtual bool do_size_checks(DCR *dcr, DEV_BLOCK *block); /* in dev.c */ + + virtual bool get_tape_alerts(DCR *dcr); + virtual void show_tape_alerts(DCR *dcr, alert_list_type type, + alert_list_which which, alert_cb alert_callback); + virtual int delete_alerts(); /* These could probably be made tape_dev only */ virtual bool bsf(int count) { return true; } virtual bool fsf(int num) { return true; } - virtual bool offline() { return true; } + virtual bool offline(DCR *dcr) { return true; } virtual void lock_door() { return; } virtual void unlock_door() { return; } - virtual bool reposition(DCR *dcr, uint32_t rfile, uint32_t rblock); + + virtual bool write_adata_label(DCR *dcr, DEV_RECORD *rec) { return false; }; + virtual void write_adata(DCR *dcr, DEV_RECORD *rec) { return; }; + virtual void write_cont_adata(DCR *dcr, DEV_RECORD *rec) { return; }; + virtual int write_adata_rechdr(DCR *dcr, DEV_RECORD *rec) { return -1; }; + virtual bool have_adata_header(DCR *dcr, DEV_RECORD *rec, + int32_t FileIndex, int32_t Stream, uint32_t VolSessionId) + { return false; }; + virtual bool read_adata_record_header(DCR *dcr, DEV_BLOCK *block, + DEV_RECORD *rec) { return false; }; + virtual void read_adata_block_header(DCR *dcr) { return; }; + virtual int read_adata(DCR *dcr, DEV_RECORD *rec) { return -1; }; + virtual void select_data_stream(DCR *dcr, DEV_RECORD *rec) { return; }; + virtual bool flush_block(DCR *dcr); /* in block_util.c */ + virtual bool do_pre_write_checks(DCR *dcr, DEV_RECORD *rec) { return true; }; + /* * Locking and blocking calls */ #ifdef DEV_DEBUG_LOCK - void dbg_Lock(const char *, int); /* in lock.c */ - void dbg_Unlock(const char *, int); /* in lock.c */ - void dbg_rLock(const char *, int, bool locked=false); /* in lock.c */ - void dbg_rUnlock(const char *, int); /* in lock.c */ + virtual void dbg_Lock(const char *, int); /* in lock.c */ + virtual void dbg_Unlock(const char *, int); /* in lock.c */ + virtual void dbg_rLock(const char *, int, bool locked=false); /* in lock.c */ + virtual void dbg_rUnlock(const char *, int); /* in lock.c */ #else - void Lock(); /* in lock.c */ - void Unlock(); /* in lock.c */ - void rLock(bool locked=false); /* in lock.c */ - void rUnlock(); /* in lock.c */ + virtual void Lock(); /* in lock.c */ + virtual void Unlock(); /* in lock.c */ + virtual void rLock(bool locked=false); /* in lock.c */ + virtual void rUnlock(); /* in lock.c */ #endif - void Lock_dcrs() { P(dcrs_mutex); }; - void Unlock_dcrs() { V(dcrs_mutex); }; #ifdef SD_DEBUG_LOCK - void dbg_Lock_acquire(const char *, int); /* in lock.c */ - void dbg_Unlock_acquire(const char *, int); /* in lock.c */ - void dbg_Lock_read_acquire(const char *, int); /* in lock.c */ - void dbg_Unlock_read_acquire(const char *, int); /* in lock.c */ - void dbg_Lock_VolCatInfo(const char *, int); /* in lock.c */ - void dbg_Unlock_VolCatInfo(const char *, int); /* in lock.c */ + virtual void dbg_Lock_acquire(const char *, int); /* in lock.c */ + virtual void dbg_Unlock_acquire(const char *, int); /* in lock.c */ + virtual void dbg_Lock_read_acquire(const char *, int); /* in lock.c */ + virtual void dbg_Unlock_read_acquire(const char *, int); /* in lock.c */ + virtual void dbg_Lock_VolCatInfo(const char *, int); /* in lock.c */ + virtual void dbg_Unlock_VolCatInfo(const char *, int); /* in lock.c */ #else - void Lock_acquire(); /* in lock.c */ - void Unlock_acquire(); /* in lock.c */ - void Lock_read_acquire(); /* in lock.c */ - void Unlock_read_acquire(); /* in lock.c */ - void Lock_VolCatInfo(); /* in lock.c */ - void Unlock_VolCatInfo(); /* in lock.c */ + virtual void Lock_acquire(); /* in lock.c */ + virtual void Unlock_acquire(); /* in lock.c */ + virtual void Lock_read_acquire(); /* in lock.c */ + virtual void Unlock_read_acquire(); /* in lock.c */ + virtual void Lock_VolCatInfo(); /* in lock.c */ + virtual void Unlock_VolCatInfo(); /* in lock.c */ #endif + virtual void dblock(int why); /* in lock.c */ + virtual void dunblock(bool locked=false); /* in lock.c */ + + int init_mutex(); /* in lock.c */ int init_acquire_mutex(); /* in lock.c */ int init_read_acquire_mutex(); /* in lock.c */ @@ -550,8 +634,8 @@ public: int init_freespace_mutex(); /* in lock.c */ void set_mutex_priorities(); /* in lock.c */ int next_vol_timedwait(const struct timespec *timeout); /* in lock.c */ - void dblock(int why); /* in lock.c */ - void dunblock(bool locked=false); /* in lock.c */ + void Lock_dcrs() { P(dcrs_mutex); }; + void Unlock_dcrs() { V(dcrs_mutex); }; bool is_device_unmounted(); /* in lock.c */ void set_blocked(int block) { m_blocked = block; }; int blocked() const { return m_blocked; }; @@ -597,13 +681,24 @@ public: dlink dev_link; /* link to attach to dev */ JCR *jcr; /* pointer to JCR */ DEVICE * volatile dev; /* pointer to device */ + DEVICE *adata_dev; /* pointer to adata dev */ DEVICE *ameta_dev; /* pointer to ameta_dev */ DEVRES *device; /* pointer to device resource */ DEV_BLOCK *block; /* pointer to block */ - DEV_BLOCK *ameta_block; /* meta data block */ + DEV_BLOCK *adata_block; /* aligned data block */ + DEV_BLOCK *ameta_block; /* aligned meta data block */ DEV_RECORD *rec; /* pointer to record */ + + VOL_LIST *CurrentVol; /* From JCR::VolList, freed at the end, passed to records */ + + alist *uploads; /* Current upload transfers to the cloud */ + alist *downloads; /* Current donwload transfers from the cloud */ + pthread_t tid; /* Thread running this dcr */ int spool_fd; /* fd if spooling */ + int crc32_type; /* Handle bad CRC32 on Solaris for volumes written with 8.4 */ + bool adata_label; /* writing adata label block */ + bool adata; /* writing aligned data block */ bool spool_data; /* set to spool data */ bool spooling; /* set when actually spooling */ bool despooling; /* set when despooling */ @@ -616,13 +711,15 @@ public: bool attached_to_dev; /* set when attached to dev */ bool keep_dcr; /* do not free dcr in release_dcr */ bool no_mount_request; /* do not submit any mount request */ + bool reading_label; /* Reading volume label */ + bool discard_invalid_records; /* we should not try to assemble invalid records */ + bool force_update_volume_info; /* update the volume information, no matter the job type */ + uint32_t VolFirstIndex; /* First file index this Volume */ uint32_t VolLastIndex; /* Last file index this Volume */ uint32_t FileIndex; /* Current File Index */ - uint32_t EndFile; /* End file written */ - uint32_t StartFile; /* Start write file */ - uint32_t StartBlock; /* Start write block */ - uint32_t EndBlock; /* Ending block written */ + uint64_t StartAddr; /* Starting write addr */ + uint64_t EndAddr; /* Ending write addr */ int64_t VolMediaId; /* MediaId */ int64_t job_spool_size; /* Current job spool size */ int64_t max_job_spool_size; /* Max job spool size */ @@ -634,10 +731,12 @@ public: int Copy; /* identical copy number */ int Stripe; /* RAIT stripe */ VOLUME_CAT_INFO VolCatInfo; /* Catalog info for desired volume */ + uint32_t crc32(unsigned char *buf, int len, uint32_t expected_crc); /* Methods */ void set_no_mount_request() { no_mount_request = true; }; /* Just fail in case of mount request */ void set_dev(DEVICE *ndev) { dev = ndev; ameta_dev = ndev; }; + void set_adata() { if (adata_dev) {dev = adata_dev; block = adata_block;} }; void set_ameta() { dev = ameta_dev; block = ameta_block; }; void set_dev_locked() { m_dev_locked = true; }; void set_writing() { m_writing = true; }; @@ -704,14 +803,11 @@ public: bool is_tape_position_ok(); /* Methods in block.c */ - void free_blocks(); bool write_block_to_device(bool final=false); bool write_block_to_dev(); bool read_block_from_device(bool check_block_numbers); bool read_block_from_dev(bool check_block_numbers); - /* Methods in label.c */ - bool rewrite_volume_label(bool recycle); }; diff --git a/bacula/src/stored/device.c b/bacula/src/stored/device.c index be01f9a409..27be8c1950 100644 --- a/bacula/src/stored/device.c +++ b/bacula/src/stored/device.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -75,9 +75,9 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) { char PrevVolName[MAX_NAME_LENGTH]; - DEV_BLOCK *label_blk; - DEV_BLOCK *block; + DEV_BLOCK *block = dcr->block; DEV_BLOCK *ameta_block = dcr->ameta_block; + DEV_BLOCK *adata_block = dcr->adata_block; char b1[30], b2[30]; time_t wait_time; char dt[MAX_TIME_LENGTH]; @@ -85,15 +85,17 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) DEVICE *dev; int blocked; /* save any previous blocked status */ bool ok = false; + bool save_adata = dcr->dev->adata; + Enter(100); + if (save_adata) { + dcr->set_ameta(); /* switch to working with ameta */ + } dev = dcr->dev; blocked = dev->blocked(); - block = dcr->block; wait_time = time(NULL); - Dmsg0(100, "=== Enter fixup_device_block_write_error\n"); - /* * If we are blocked at entry, unblock it, and set our own block status */ @@ -108,8 +110,7 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) bstrncpy(PrevVolName, dev->getVolCatName(), sizeof(PrevVolName)); bstrncpy(dev->VolHdr.PrevVolumeName, PrevVolName, sizeof(dev->VolHdr.PrevVolumeName)); - label_blk = new_block(dev); - dcr->ameta_block = dcr->block = label_blk; + dev->new_dcr_blocks(dcr); /* Inform User about end of medium */ Jmsg(jcr, M_INFO, 0, _("End of medium on Volume \"%s\" Bytes=%s Blocks=%s at %s.\n"), @@ -121,13 +122,16 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) dev->set_unload(); /* Clear DCR Start/End Block/File positions */ - dcr->StartBlock = dcr->EndBlock = 0; - dcr->StartFile = dcr->EndFile = 0; + dcr->VolFirstIndex = dcr->VolLastIndex = 0; + dcr->StartAddr = dcr->EndAddr = 0; + dcr->VolMediaId = 0; + dcr->WroteVol = false; if (!dcr->mount_next_write_volume()) { - free_block(label_blk); + dev->free_dcr_blocks(dcr); dcr->block = block; dcr->ameta_block = ameta_block; + dcr->adata_block = adata_block; dev->Lock(); goto bail_out; } @@ -155,14 +159,16 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) berrno be; Pmsg1(0, _("write_block_to_device Volume label failed. ERR=%s"), be.bstrerror(dev->dev_errno)); - free_block(label_blk); + dev->free_dcr_blocks(dcr); dcr->block = block; dcr->ameta_block = ameta_block; + dcr->adata_block = adata_block; goto bail_out; } - free_block(label_blk); + dev->free_dcr_blocks(dcr); dcr->block = block; dcr->ameta_block = ameta_block; + dcr->adata_block = adata_block; /* Clear NewVol now because dir_get_volume_info() already done */ jcr->dcr->NewVol = false; @@ -172,6 +178,9 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) /* Write overflow block to device */ Dmsg0(190, "Write overflow block to dev\n"); + if (save_adata) { + dcr->set_adata(); /* try to write block we entered with */ + } if (!dcr->write_block_to_dev()) { berrno be; Dmsg1(0, _("write_block_to_device overflow block failed. ERR=%s"), @@ -187,6 +196,9 @@ bool fixup_device_block_write_error(DCR *dcr, int retries) ok = true; bail_out: + if (save_adata) { + dcr->set_ameta(); /* Do unblock ... on ameta */ + } /* * At this point, the device is locked and blocked. * Unblock the device, restore any entry blocked condition, then @@ -196,6 +208,9 @@ bail_out: if (blocked != BST_NOT_BLOCKED) { block_device(dev, blocked); } + if (save_adata) { + dcr->set_adata(); /* switch back to what we entered with */ + } return ok; /* device locked */ } @@ -204,11 +219,17 @@ void set_start_vol_position(DCR *dcr) DEVICE *dev = dcr->dev; /* Set new start position */ if (dev->is_tape()) { - dcr->StartBlock = dcr->EndBlock = dev->block_num; - dcr->StartFile = dcr->EndFile = dev->file; + dcr->StartAddr = dcr->EndAddr = dev->get_full_addr(); } else { - dcr->StartBlock = dcr->EndBlock = (uint32_t)dev->file_addr; - dcr->StartFile = dcr->EndFile = (uint32_t)(dev->file_addr >> 32); + if (dev->adata) { + dev = dcr->ameta_dev; + } + /* + * Note: we only update the DCR values for ameta blocks + * because all the indexing (JobMedia) is done with + * ameta blocks/records, which may point to adata. + */ + dcr->StartAddr = dcr->EndAddr = dev->get_full_addr(); } } @@ -226,7 +247,7 @@ void set_new_volume_parameters(DCR *dcr) int retries = 5; wait_for_device(dcr, retries); } - if (dir_get_volume_info(dcr, GET_VOL_INFO_FOR_WRITE)) { + if (dir_get_volume_info(dcr, dcr->VolumeName, GET_VOL_INFO_FOR_WRITE)) { dcr->dev->clear_wait(); } else { Dmsg1(40, "getvolinfo failed. No new Vol: %s", jcr->errmsg); @@ -289,15 +310,9 @@ bool first_open_device(DCR *dcr) goto bail_out; } - int mode; - if (dev->has_cap(CAP_STREAM)) { - mode = OPEN_WRITE_ONLY; - } else { - mode = OPEN_READ_ONLY; - } Dmsg0(129, "Opening device.\n"); - if (!dev->open(dcr, mode)) { - Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), dev->errmsg); + if (!dev->open_device(dcr, OPEN_READ_ONLY)) { + Jmsg1(NULL, M_FATAL, 0, _("dev open failed: %s\n"), dev->errmsg); ok = false; goto bail_out; } @@ -307,31 +322,3 @@ bail_out: dev->rUnlock(); return ok; } - -/* - * Make sure device is open, if not do so - */ -bool open_device(DCR *dcr) -{ - DEVICE *dev = dcr->dev; - /* Open device */ - int mode; - if (dev->has_cap(CAP_STREAM)) { - mode = OPEN_WRITE_ONLY; - } else { - mode = OPEN_READ_WRITE; - } - if (!dev->open(dcr, mode)) { - /* If polling, ignore the error */ - /* If DVD, also ignore the error, very often you cannot open the device - * (when there is no DVD, or when the one inserted is a wrong one) */ - if (!dev->poll && !dev->is_dvd() && !dev->is_removable()) { - Jmsg2(dcr->jcr, M_FATAL, 0, _("Unable to open device %s: ERR=%s\n"), - dev->print_name(), dev->bstrerror()); - Pmsg2(000, _("Unable to open archive %s: ERR=%s\n"), - dev->print_name(), dev->bstrerror()); - } - return false; - } - return true; -} diff --git a/bacula/src/stored/dircmd.c b/bacula/src/stored/dircmd.c index 4714834fa7..990a2febe0 100644 --- a/bacula/src/stored/dircmd.c +++ b/bacula/src/stored/dircmd.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -69,6 +69,8 @@ static bool storage_cmd(JCR *jcr); static bool label_cmd(JCR *jcr); static bool die_cmd(JCR *jcr); static bool relabel_cmd(JCR *jcr); +static bool truncate_cache_cmd(JCR *jcr); +static bool upload_cmd(JCR *jcr); static bool readlabel_cmd(JCR *jcr); static bool release_cmd(JCR *jcr); static bool setdebug_cmd(JCR *jcr); @@ -79,10 +81,14 @@ static bool enable_cmd(JCR *jcr); static bool disable_cmd(JCR *jcr); //static bool action_on_purge_cmd(JCR *jcr); static bool bootstrap_cmd(JCR *jcr); +static bool cloud_list_cmd(JCR *jcr); +static bool cloud_prunecache_cmd(JCR *jcr); static bool changer_cmd(JCR *sjcr); static bool do_label(JCR *jcr, int relabel); static DCR *find_device(JCR *jcr, POOL_MEM &dev_name, POOLMEM *media_type, int drive); +static DCR *find_any_device(JCR *jcr, POOL_MEM &dev_name, + POOLMEM *media_type, int drive); static void read_volume_label(JCR *jcr, DCR *dcr, DEVICE *dev, int Slot); static void label_volume_if_ok(DCR *dcr, char *oldname, char *newname, char *poolname, @@ -115,8 +121,8 @@ static struct s_cmds cmds[] = { {".die", die_cmd, 0}, {"label", label_cmd, 0}, /* label a tape */ {"mount", mount_cmd, 0}, - {"enable", enable_cmd, 0}, - {"disable", disable_cmd, 0}, + {"enable", enable_cmd, 0}, + {"disable", disable_cmd, 0}, {"readlabel", readlabel_cmd, 0}, {"release", release_cmd, 0}, {"relabel", relabel_cmd, 0}, /* relabel a tape */ @@ -125,6 +131,10 @@ static struct s_cmds cmds[] = { {".status", qstatus_cmd, 1}, {"stop", cancel_cmd, 0}, {"storage", storage_cmd, 0}, /* get SD addr from Dir */ + {"truncate", truncate_cache_cmd, 0}, + {"upload", upload_cmd, 0}, + {"prunecache", cloud_prunecache_cmd, 0}, + {"cloudlist", cloud_list_cmd, 0}, /* List volumes/parts in the cloud */ {"unmount", unmount_cmd, 0}, {"use storage=", use_cmd, 0}, {"run", run_cmd, 0}, @@ -198,17 +208,19 @@ void *handle_connection_request(void *arg) goto bail_out; } if (!authenticate_director(jcr)) { - Jmsg(jcr, M_FATAL, 0, _("Unable to authenticate Director\n")); + Jmsg(jcr, M_FATAL, 0, _("[SF0100] Unable to authenticate Director\n")); goto bail_out; } Dmsg0(90, "Message channel init completed.\n"); + dequeue_messages(jcr); /* dequeue any daemon messages */ + for (quit=false; !quit;) { /* Read command */ if ((bnet_stat = bs->recv()) <= 0) { break; /* connection terminated */ } - Dmsg1(199, "msg); + Dmsg1(199, "msg); /* Ensure that device initialization is complete */ while (!init_done) { bmicrosleep(1, 0); @@ -293,7 +305,7 @@ static bool client_cmd(JCR *jcr) if (sscanf(dir->msg, "client address=%s port=%d ssl=%d", &jcr->client_addr, &client_port, &enable_ssl) != 3) { pm_strcpy(jcr->errmsg, dir->msg); - Jmsg(jcr, M_FATAL, 0, _("Bad client command: %s"), jcr->errmsg); + Jmsg(jcr, M_FATAL, 0, _("[SF0101] Bad client command: %s"), jcr->errmsg); Dmsg1(050, "Bad client command: %s", jcr->errmsg); goto bail_out; } @@ -306,7 +318,7 @@ static bool client_cmd(JCR *jcr) _("Client daemon"), jcr->client_addr, NULL, client_port, 1)) { /* destroy() OK because cl is local */ cl->destroy(); - Jmsg(jcr, M_FATAL, 0, _("Failed to connect to Client daemon: %s:%d\n"), + Jmsg(jcr, M_FATAL, 0, _("[SF0102] Failed to connect to Client daemon: %s:%d\n"), jcr->client_addr, client_port); Dmsg2(100, "Failed to connect to Client daemon: %s:%d\n", jcr->client_addr, client_port); @@ -345,7 +357,7 @@ static bool storage_cmd(JCR *jcr) if (sscanf(dir->msg, storaddr, &jcr->stored_addr, &stored_port, &enable_ssl, Job, sd_auth_key) != 5) { pm_strcpy(jcr->errmsg, dir->msg); - Jmsg(jcr, M_FATAL, 0, _("Bad storage command: %s"), jcr->errmsg); + Jmsg(jcr, M_FATAL, 0, _("[SF0103] Bad storage command: %s"), jcr->errmsg); Pmsg1(010, "Bad storage command: %s", jcr->errmsg); goto bail_out; } @@ -367,7 +379,7 @@ static bool storage_cmd(JCR *jcr) _("Storage daemon"), jcr->stored_addr, NULL, stored_port, 1)) { /* destroy() OK because sd is local */ sd->destroy(); - Jmsg(jcr, M_FATAL, 0, _("Failed to connect to Storage daemon: %s:%d\n"), + Jmsg(jcr, M_FATAL, 0, _("[SF0104] Failed to connect to Storage daemon: %s:%d\n"), jcr->stored_addr, stored_port); Dmsg2(010, "Failed to connect to Storage daemon: %s:%d\n", jcr->stored_addr, stored_port); @@ -384,7 +396,7 @@ static bool storage_cmd(JCR *jcr) jcr->store_bsock = jcr->file_bsock; } if (jcr->store_bsock == NULL) { - Jmsg0(jcr, M_FATAL, 0, _("In storage_cmd port==0, no prior Storage connection.\n")); + Jmsg0(jcr, M_FATAL, 0, _("[SF0105] In storage_cmd port==0, no prior Storage connection.\n")); Pmsg0(010, "In storage_cmd port==0, no prior Storage connection.\n"); goto bail_out; } @@ -453,6 +465,19 @@ static bool setdebug_cmd(JCR *jcr) } debug_level_tags = level_tags; + /* TODO: Temp code to activate the new BSR optimisation code */ + for (char *p = options; *p ; p++) { + switch(*p) { + case 'i': /* Use new match_bsr() code */ + use_new_match_all = 1; + break; + case '0': + use_new_match_all = 0; + break; + } + } + /* **** */ + return dir->fsend(OKsetdebug, lvl, trace_flag, options, tags); } @@ -587,16 +612,20 @@ static bool do_label(JCR *jcr, int relabel) ok = false; } if (!ok) { - Dmsg2(000, "Reserve error on volume \"%s\": ERR=%s\n", newname, jcr->errmsg); dir->fsend(_("3908 Error reserving volume: %s\n"), jcr->errmsg); dev->max_concurrent_jobs = max_jobs; dev->Unlock(); goto bail_out; } + + /* some command use recv and don't accept catalog update. + * it's not the case here, so we force dir_update_volume_info catalog update */ + dcr->force_update_volume_info = true; + if (!dev->is_open() && !dev->is_busy()) { Dmsg1(400, "Can %slabel. Device is not open\n", relabel?"re":""); label_volume_if_ok(dcr, oldname, newname, poolname, slot, relabel); - dev->close(); + dev->close(dcr); /* Under certain "safe" conditions, we can steal the lock */ } else if (dev->can_steal_lock()) { Dmsg0(400, "Can relabel. can_steal_lock\n"); @@ -610,6 +639,13 @@ static bool do_label(JCR *jcr, int relabel) dev->max_concurrent_jobs = max_jobs; volume_unused(dcr); dev->Unlock(); +#ifdef DEVELOPER + if (chk_dbglvl(DT_VOLUME)) { + Dmsg0(0, "Waiting few seconds to force a bug...\n"); + bmicrosleep(30, 0); + Dmsg0(0, "Doing free_volume()\n"); + } +#endif } else { dir->fsend(_("3999 Device \"%s\" not found or could not be opened.\n"), dev_name.c_str()); } @@ -630,6 +666,233 @@ bail_out: return true; } +/* + * Handles truncate cache commands + */ +static bool truncate_cache_cmd(JCR *jcr) +{ + POOLMEM *volname, *poolname, *mtype; + POOL_MEM dev_name; + BSOCK *dir = jcr->dir_bsock; + DCR *dcr = NULL;; + DEVICE *dev; + bool ok = false; + int32_t slot, drive; + int nbpart = 0; + int64_t size = 0; + char ed1[50]; + + volname = get_memory(dir->msglen+1); + poolname = get_memory(dir->msglen+1); + mtype = get_memory(dir->msglen+1); + if (sscanf(dir->msg, "truncate cache Storage=%127s Volume=%127s PoolName=%127s " + "MediaType=%127s Slot=%d drive=%d", + dev_name.c_str(), volname, poolname, mtype, + &slot, &drive) == 6) { + ok = true; + } + if (ok) { + unbash_spaces(volname); + unbash_spaces(poolname); + unbash_spaces(mtype); + dcr = find_device(jcr, dev_name, mtype, drive); + if (dcr) { + uint32_t max_jobs; + dev = dcr->dev; + ok = true; + dev->Lock(); /* Use P to avoid indefinite block */ + max_jobs = dev->max_concurrent_jobs; + dev->max_concurrent_jobs = 1; + bstrncpy(dcr->VolumeName, volname, sizeof(dcr->VolumeName)); + if (dcr->can_i_write_volume()) { + if (reserve_volume(dcr, volname) == NULL) { + ok = false; + } + Dmsg1(400, "Reserved volume \"%s\"\n", volname); + } else { + ok = false; + } + if (!ok) { + dir->fsend(_("3908 Error reserving volume \"%s\": %s\n"), volname, jcr->errmsg); + dev->max_concurrent_jobs = max_jobs; + dev->Unlock(); + goto bail_out; + } + if ((!dev->is_open() && !dev->is_busy()) || dev->can_steal_lock()) { + Dmsg0(400, "Call truncate_cache\n"); + nbpart = dev->truncate_cache(dcr, volname, &size); + if (nbpart >= 0) { + dir->fsend(_("3000 OK truncate cache for volume \"%s\" %d part(s) %sB\n"), + volname, nbpart, edit_uint64_with_suffix(size, ed1)); + } else { + dir->fsend(_("3900 Truncate cache for volume \"%s\" failed. ERR=%s\n"), volname, dev->errmsg); + } + } else if (dev->is_busy() || dev->is_blocked()) { + send_dir_busy_message(dir, dev); + } else { /* device not being used */ + Dmsg0(400, "Call truncate_cache\n"); + nbpart = dev->truncate_cache(dcr, volname, &size); + if (nbpart >= 0) { + dir->fsend(_("3000 OK truncate cache for volume \"%s\" %d part(s) %sB\n"), volname, + nbpart, edit_uint64_with_suffix(size, ed1)); + } else { + dir->fsend(_("3900 Truncate cache for volume \"%s\" failed. ERR=%s\n"), volname, dev->errmsg); + } + } + dev->max_concurrent_jobs = max_jobs; + volume_unused(dcr); + dev->Unlock(); +#ifdef DEVELOPER + if (chk_dbglvl(DT_VOLUME)) { + Dmsg0(0, "Waiting few seconds to force a bug...\n"); + bmicrosleep(30, 0); + Dmsg0(0, "Doing free_volume()\n"); + } +#endif + } else { + dir->fsend(_("3999 Device \"%s\" not found or could not be opened.\n"), dev_name.c_str()); + } + } else { + /* NB dir->msg gets clobbered in bnet_fsend, so save command */ + pm_strcpy(jcr->errmsg, dir->msg); + dir->fsend(_("3911 Error scanning truncate command: %s\n"), jcr->errmsg); + } +bail_out: + if (dcr) { + free_dcr(dcr); + } + free_memory(volname); + free_memory(poolname); + free_memory(mtype); + dir->signal(BNET_EOD); + return true; +} + +static bool cloud_prunecache_cmd(JCR *jcr) +{ + /* TODO: Implement a function to prune the cache of a cloud device */ + jcr->dir_bsock->fsend(_("3900 Not yet implemented\n")); + return true; +} + +/* List volumes in the cloud */ +static bool cloud_list_cmd(JCR *jcr) +{ + jcr->dir_bsock->fsend(_("3900 Not yet implemented\n")); + return true; +} + +/* + * Handles upload cache to Cloud command + */ +static bool upload_cmd(JCR *jcr) +{ + POOLMEM *volname, *poolname, *mtype, *err; + POOL_MEM dev_name; + BSOCK *dir = jcr->dir_bsock; + DCR *dcr = NULL; + DEVICE *dev; + bool ok = false; + int32_t slot, drive; + + volname = get_memory(dir->msglen+1); + poolname = get_memory(dir->msglen+1); + mtype = get_memory(dir->msglen+1); + err = get_pool_memory(PM_MESSAGE); + *volname = *poolname = *mtype = *err = 0; + + if (sscanf(dir->msg, "upload Storage=%127s Volume=%127s PoolName=%127s " + "MediaType=%127s Slot=%d drive=%d", + dev_name.c_str(), volname, poolname, mtype, + &slot, &drive) == 6) { + ok = true; + } + if (ok) { + unbash_spaces(volname); + unbash_spaces(poolname); + unbash_spaces(mtype); + dcr = find_device(jcr, dev_name, mtype, drive); + if (dcr) { + uint32_t max_jobs; + dev = dcr->dev; + ok = true; + dev->Lock(); /* Use P to avoid indefinite block */ + max_jobs = dev->max_concurrent_jobs; + dev->max_concurrent_jobs = 1; + bstrncpy(dcr->VolumeName, volname, sizeof(dcr->VolumeName)); + if (dcr->can_i_write_volume()) { + if (reserve_volume(dcr, volname) == NULL) { + ok = false; + } + Dmsg1(400, "Reserved volume \"%s\"\n", volname); + } else { + ok = false; + } + if (!ok) { + dir->fsend(_("3908 Error reserving volume \"%s\": %s\n"), volname, jcr->errmsg); + dev->max_concurrent_jobs = max_jobs; + dev->Unlock(); + goto bail_out; + } + if ((!dev->is_open() && !dev->is_busy()) || dev->can_steal_lock()) { + Dmsg0(400, "Can upload, because device is not open.\n"); + dev->setVolCatName(volname); + dev->part = 0; + if (dev->open_device(dcr, OPEN_READ_WRITE)) { + ok = dev->upload_cache(dcr, volname, err); + dev->part = 0; + dev->close(dcr); + dev->end_of_job(dcr); + } + } else if (dev->is_busy() || dev->is_blocked()) { + send_dir_busy_message(dir, dev); + } else { /* device not being used */ + Dmsg0(400, "Can upload, because device not used\n"); + dev->setVolCatName(volname); + dev->part = 0; + if (dev->open_device(dcr, OPEN_READ_WRITE)) { + ok = dev->upload_cache(dcr, volname, err); + dev->part = 0; + dev->close(dcr); + dev->end_of_job(dcr); + } + } + dev->max_concurrent_jobs = max_jobs; + volume_unused(dcr); + dev->Unlock(); +#ifdef DEVELOPER + if (chk_dbglvl(DT_VOLUME)) { + Dmsg0(0, "Waiting few seconds to force a bug...\n"); + bmicrosleep(30, 0); + Dmsg0(0, "Doing free_volume()\n"); + } +#endif + } else { + dir->fsend(_("3999 Device \"%s\" not found or could not be opened.\n"), dev_name.c_str()); + } + } else { + /* NB dir->msg gets clobbered in bnet_fsend, so save command */ + pm_strcpy(jcr->errmsg, dir->msg); + dir->fsend(_("3912 Error scanning upload command: ERR=%s\n"), jcr->errmsg); + } +bail_out: + if (ok) { + dir->fsend(_("3000 OK upload.\n")); + } else { + dir->fsend(_("3999 Error with the upload: ERR=%s\n"), err); + } + if (dcr) { + free_dcr(dcr); + } + free_pool_memory(err); + free_memory(volname); + free_memory(poolname); + free_memory(mtype); + dir->signal(BNET_EOD); + return true; +} + + /* * Read the tape label and determine if we can safely * label the tape (not a Bacula volume), then label it. @@ -646,6 +909,7 @@ static void label_volume_if_ok(DCR *dcr, char *oldname, int label_status; int mode; const char *volname = (relabel == 1) ? oldname : newname; + uint64_t volCatBytes; steal_device_lock(dev, &hold, BST_WRITING_LABEL); Dmsg1(100, "Stole device %s lock, writing label.\n", dev->print_name()); @@ -655,26 +919,27 @@ static void label_volume_if_ok(DCR *dcr, char *oldname, goto bail_out; /* error */ } + + if (relabel) { + dev->truncating = true; /* let open_device() know we will truncate it */ + } + /* Set old volume name for open if relabeling */ + dcr->setVolCatName(volname); + /* Ensure that the device is open -- autoload_device() closes it */ if (dev->is_tape()) { mode = OPEN_READ_WRITE; } else { mode = CREATE_READ_WRITE; } - - if (relabel) { - dev->truncating = true; /* let open() know we will truncate it */ - } - /* Set old volume name for open if relabeling */ - dcr->setVolCatName(volname); - if (!dev->open(dcr, mode)) { - dir->fsend(_("3910 Unable to open device \"%s\": ERR=%s\n"), + if (!dev->open_device(dcr, mode)) { + dir->fsend(_("3929 Unable to open device \"%s\": ERR=%s\n"), dev->print_name(), dev->bstrerror()); goto bail_out; } /* See what we have for a Volume */ - label_status = read_dev_volume_label(dcr); + label_status = dev->read_dev_volume_label(dcr); /* Set new volume name */ dcr->setVolCatName(newname); @@ -702,39 +967,55 @@ static void label_volume_if_ok(DCR *dcr, char *oldname, /* Fall through wanted! */ case VOL_IO_ERROR: case VOL_NO_LABEL: - if (!write_new_volume_label_to_dev(dcr, newname, poolname, - relabel, true /* write dvd now */)) { + if (!dev->write_volume_label(dcr, newname, poolname, + relabel, true /* write label now */)) { dir->fsend(_("3912 Failed to label Volume: ERR=%s\n"), dcr->jcr->errmsg); break; } + volCatBytes = dev->VolCatInfo.VolCatBytes; + /* + * After writing label, create a new part + */ + if (dev->is_cloud()) { + dev->set_append(); + if (!dev->open_next_part(dcr)) { + dir->fsend(_("3913 Failed to open next part: ERR=%s\n"), dcr->jcr->errmsg); + break; + } + } bstrncpy(dcr->VolumeName, newname, sizeof(dcr->VolumeName)); /* The following 3000 OK label. string is scanned in ua_label.c */ int type; - if (dev->dev_type == B_FILE_DEV) { + if (dev->dev_type == B_FILE_DEV || dev->dev_type == B_ALIGNED_DEV || + dev->dev_type == B_CLOUD_DEV) + { type = dev->dev_type; } else { type = 0; } dir->fsend("3000 OK label. VolBytes=%lld VolABytes=%lld VolType=%d Volume=\"%s\" Device=%s\n", - dev->VolCatInfo.VolCatBytes, dev->VolCatInfo.VolCatAdataBytes, + volCatBytes, dev->VolCatInfo.VolCatAdataBytes, type, newname, dev->print_name()); break; case VOL_TYPE_ERROR: - dir->fsend(_("3912 Failed to label Volume: ERR=%s\n"), dcr->jcr->errmsg); + dir->fsend(_("3917 Failed to label Volume: ERR=%s\n"), dcr->jcr->errmsg); break; case VOL_NO_MEDIA: - dir->fsend(_("3914 Failed to label Volume (no media): ERR=%s\n"), dcr->jcr->errmsg); + dir->fsend(_("3918 Failed to label Volume (no media): ERR=%s\n"), dcr->jcr->errmsg); break; default: - dir->fsend(_("3913 Cannot label Volume. " + dir->fsend(_("3919 Cannot label Volume. " "Unknown status %d from read_volume_label()\n"), label_status); break; } bail_out: if (dev->is_open() && !dev->has_cap(CAP_ALWAYSOPEN)) { - dev->close(); + dev->close(dcr); } + + dev->end_of_job(dcr); + if (!dev->is_open()) { dev->clear_volhdr(); } @@ -761,7 +1042,7 @@ static bool read_label(DCR *dcr) dcr->VolumeName[0] = 0; dev->clear_labeled(); /* force read of label */ - switch (read_dev_volume_label(dcr)) { + switch (dev->read_dev_volume_label(dcr)) { case VOL_OK: dir->fsend(_("3001 Mounted Volume: %s\n"), dev->VolHdr.VolumeName); ok = true; @@ -799,7 +1080,7 @@ static DCR *find_device(JCR *jcr, POOL_MEM &devname, } if (!device->dev) { Jmsg(jcr, M_WARNING, 0, _("\n" - " Device \"%s\" requested by DIR could not be opened or does not exist.\n"), + "[SW0106] Device \"%s\" requested by DIR could not be opened or does not exist.\n"), devname.c_str()); continue; } @@ -821,7 +1102,7 @@ static DCR *find_device(JCR *jcr, POOL_MEM &devname, if (!device->dev) { Dmsg1(100, "Device %s could not be opened. Skipped\n", devname.c_str()); Jmsg(jcr, M_WARNING, 0, _("\n" - " Device \"%s\" in changer \"%s\" requested by DIR could not be opened or does not exist.\n"), + "[SW0107] Device \"%s\" in changer \"%s\" requested by DIR could not be opened or does not exist.\n"), device->hdr.name, devname.c_str()); continue; } @@ -854,6 +1135,78 @@ static DCR *find_device(JCR *jcr, POOL_MEM &devname, return dcr; } +/* + * Find even disabled devices so that we can enable them + * ***FIXME*** This could probably be merged with find_device with another + * argument, but this is easier for the moment. + */ +static DCR *find_any_device(JCR *jcr, POOL_MEM &devname, + POOLMEM *media_type, int drive) +{ + DEVRES *device; + AUTOCHANGER *changer; + bool found = false; + DCR *dcr = NULL; + + unbash_spaces(devname); + foreach_res(device, R_DEVICE) { + /* Find resource, and make sure we were able to open it */ + if (strcmp(device->hdr.name, devname.c_str()) == 0 && + (!media_type || strcmp(device->media_type, media_type) ==0)) { + if (!device->dev) { + device->dev = init_dev(jcr, device); + } + if (!device->dev) { + Jmsg(jcr, M_WARNING, 0, _("\n" + "[SW0108] Device \"%s\" requested by DIR could not be opened or does not exist.\n"), + devname.c_str()); + continue; + } + Dmsg1(20, "Found device %s\n", device->hdr.name); + found = true; + break; + } + } + if (!found) { + foreach_res(changer, R_AUTOCHANGER) { + /* Find resource, and make sure we were able to open it */ + if (strcmp(devname.c_str(), changer->hdr.name) == 0) { + /* Try each device in this AutoChanger */ + foreach_alist(device, changer->device) { + Dmsg1(100, "Try changer device %s\n", device->hdr.name); + if (!device->dev) { + device->dev = init_dev(jcr, device); + } + if (!device->dev) { + Dmsg1(100, "Device %s could not be opened. Skipped\n", devname.c_str()); + Jmsg(jcr, M_WARNING, 0, _("\n" + "[SW0109] Device \"%s\" in changer \"%s\" requested by DIR could not be opened or does not exist.\n"), + device->hdr.name, devname.c_str()); + continue; + } + if ((drive < 0 || drive == (int)device->dev->drive_index) && + (!media_type || strcmp(device->media_type, media_type) ==0)) { + Dmsg1(20, "Found changer device %s\n", device->hdr.name); + found = true; + break; + } + Dmsg3(100, "Device %s drive wrong: want=%d got=%d skipping\n", + devname.c_str(), drive, (int)device->dev->drive_index); + } + break; /* we found it but could not open a device */ + } + } + } + + if (found) { + Dmsg1(100, "Found device %s\n", device->hdr.name); + dcr = new_dcr(jcr, NULL, device->dev); + dcr->device = device; + } + return dcr; +} + + /* * Mount command from Director */ @@ -870,10 +1223,6 @@ static bool mount_cmd(JCR *jcr) Dmsg1(100, "%s\n", dir->msg); ok = sscanf(dir->msg, "mount %127s drive=%d slot=%d", devname.c_str(), &drive, &slot) == 3; - if (!ok) { - slot = 0; - ok = sscanf(dir->msg, "mount %127s drive=%d", devname.c_str(), &drive) == 2; - } Dmsg3(100, "ok=%d device_index=%d slot=%d\n", ok, drive, slot); if (ok) { dcr = find_device(jcr, devname, NULL, drive); @@ -904,7 +1253,7 @@ static bool mount_cmd(JCR *jcr) try_autoload_device(jcr, dcr, slot, ""); } /* We freed the device, so reopen it and wake any waiting threads */ - if (!dev->open(dcr, OPEN_READ_ONLY)) { + if (!dev->open_device(dcr, OPEN_READ_ONLY)) { dir->fsend(_("3901 Unable to open device \"%s\": ERR=%s\n"), dev->print_name(), dev->bstrerror()); if (dev->blocked() == BST_UNMOUNTED) { @@ -914,7 +1263,7 @@ static bool mount_cmd(JCR *jcr) } break; } - read_dev_volume_label(dcr); + dev->read_dev_volume_label(dcr); if (dev->blocked() == BST_UNMOUNTED) { /* We blocked the device, so unblock it */ Dmsg0(100, "Unmounted. Unblocking device\n"); @@ -962,7 +1311,7 @@ static bool mount_cmd(JCR *jcr) dev->print_name()); } } else if (dev->is_tape()) { - if (!dev->open(dcr, OPEN_READ_ONLY)) { + if (!dev->open_device(dcr, OPEN_READ_ONLY)) { dir->fsend(_("3901 Unable to open device \"%s\": ERR=%s\n"), dev->print_name(), dev->bstrerror()); break; @@ -977,7 +1326,7 @@ static bool mount_cmd(JCR *jcr) dev->print_name()); } if (dev->is_open() && !dev->has_cap(CAP_ALWAYSOPEN)) { - dev->close(); + dev->close(dcr); } } else if (dev->is_unmountable()) { if (dev->mount(1)) { @@ -1024,17 +1373,27 @@ static bool enable_cmd(JCR *jcr) DCR *dcr; int32_t drive; bool ok; + int deleted; ok = sscanf(dir->msg, "enable %127s drive=%d", devname.c_str(), &drive) == 2; Dmsg3(100, "ok=%d device=%s device_index=%d\n", ok, devname.c_str(), drive); if (ok) { - dcr = find_device(jcr, devname, NULL, drive); + dcr = find_any_device(jcr, devname, NULL, drive); if (dcr) { dev = dcr->dev; dev->Lock(); /* Use P to avoid indefinite block */ - dev->enabled = true; - dir->fsend(_("3002 Device \"%s\" enabled.\n"), dev->print_name()); + if (dev->enabled) { + dir->fsend(_("3003 Device \"%s\" already enabled.\n"), dev->print_name()); + } else { + dev->enabled = true; + dir->fsend(_("3002 Device \"%s\" enabled.\n"), dev->print_name()); + } + deleted = dev->delete_alerts(); + if (deleted > 0) { + dir->fsend(_("3004 Device \"%s\" deleted %d alert%s.\n"), + dev->print_name(), deleted, deleted>1?"s":""); + } dev->Unlock(); free_dcr(dcr); } @@ -1120,7 +1479,7 @@ static bool unmount_cmd(JCR *jcr) * ***FIXME**** what is this ???? -- probably we had * the wrong volume so we must free it and try again. KES */ - dev->close(); + dev->close(dcr); free_volume(dev); } if (dev->is_unmountable() && !dev->unmount(0)) { @@ -1152,7 +1511,7 @@ static bool unmount_cmd(JCR *jcr) dev->set_blocked(BST_UNMOUNTED); clear_thread_id(dev->no_wait_id); if (!unload_autochanger(dcr, -1)) { - dev->close(); + dev->close(dcr); free_volume(dev); } if (dev->is_unmountable() && !dev->unmount(0)) { @@ -1307,10 +1666,10 @@ static bool get_bootstrap_file(JCR *jcr, BSOCK *sock) V(bsr_mutex); Dmsg1(400, "bootstrap=%s\n", fname); jcr->RestoreBootstrap = fname; - bs = fopen(fname, "a+b"); /* create file */ + bs = bfopen(fname, "a+b"); /* create file */ if (!bs) { berrno be; - Jmsg(jcr, M_FATAL, 0, _("Could not create bootstrap file %s: ERR=%s\n"), + Jmsg(jcr, M_FATAL, 0, _("[SF0110] Could not create bootstrap file %s: ERR=%s\n"), jcr->RestoreBootstrap, be.bstrerror()); goto bail_out; } @@ -1323,14 +1682,15 @@ static bool get_bootstrap_file(JCR *jcr, BSOCK *sock) Dmsg0(150, "=== end bootstrap file ===\n"); jcr->bsr = parse_bsr(jcr, jcr->RestoreBootstrap); if (!jcr->bsr) { - Jmsg(jcr, M_FATAL, 0, _("Error parsing bootstrap file.\n")); + Jmsg(jcr, M_FATAL, 0, _("[SF0111] Error parsing bootstrap file.\n")); goto bail_out; } if (chk_dbglvl(150)) { - dump_bsr(jcr->bsr, true); + dump_bsr(NULL, jcr->bsr, true); } + /* If we got a bootstrap, we are reading, so create read volume list */ - create_restore_volume_list(jcr); + create_restore_volume_list(jcr, true /* store the volumes in the global vol_read list */); ok = true; bail_out: @@ -1428,7 +1788,7 @@ static bool readlabel_cmd(JCR *jcr) dev->Lock(); /* Use P to avoid indefinite block */ if (!dev->is_open()) { read_volume_label(jcr, dcr, dev, Slot); - dev->close(); + dev->close(dcr); /* Under certain "safe" conditions, we can steal the lock */ } else if (dev->can_steal_lock()) { read_volume_label(jcr, dcr, dev, Slot); @@ -1469,7 +1829,7 @@ static void read_volume_label(JCR *jcr, DCR *dcr, DEVICE *dev, int Slot) } dev->clear_labeled(); /* force read of label */ - switch (read_dev_volume_label(dcr)) { + switch (dev->read_dev_volume_label(dcr)) { case VOL_OK: /* DO NOT add quotes around the Volume name. It is scanned in the DIR */ dir->fsend(_("3001 Volume=%s Slot=%d\n"), dev->VolHdr.VolumeName, Slot); diff --git a/bacula/src/stored/fd_cmds.c b/bacula/src/stored/fd_cmds.c index 1b6bfe8787..c8ad493f75 100644 --- a/bacula/src/stored/fd_cmds.c +++ b/bacula/src/stored/fd_cmds.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -26,8 +26,6 @@ * then when the Storage daemon receives a proper connection from * the File daemon, control is passed here to handle the * subsequent File daemon commands. - * - * */ #include "bacula.h" @@ -41,6 +39,7 @@ extern STORES *me; /* Static variables */ static char ferrmsg[] = "3900 Invalid command\n"; +static char OK_data[] = "3000 OK data\n"; /* Imported functions */ extern bool do_append_data(JCR *jcr); @@ -55,6 +54,8 @@ static bool append_end_session(JCR *jcr); static bool read_open_session(JCR *jcr); static bool read_data_cmd(JCR *jcr); static bool read_close_session(JCR *jcr); +static bool read_control_cmd(JCR *jcr); +static bool sd_testnetwork_cmd(JCR *jcr); /* Exported function */ bool get_bootstrap_file(JCR *jcr, BSOCK *bs); @@ -75,6 +76,8 @@ static struct s_cmds fd_cmds[] = { {"read open", read_open_session}, {"read data", read_data_cmd}, {"read close", read_close_session}, + {"read control", read_control_cmd}, + {"testnetwork", sd_testnetwork_cmd}, {NULL, NULL} /* list terminator */ }; @@ -93,7 +96,7 @@ static char ERROR_append[] = "3903 Error append data: %s\n"; /* Information sent to the Director */ static char Job_start[] = "3010 Job %s start\n"; char Job_end[] = - "3099 Job %s end JobStatus=%d JobFiles=%d JobBytes=%s JobErrors=%u\n"; + "3099 Job %s end JobStatus=%d JobFiles=%d JobBytes=%s JobErrors=%u ErrMsg=%s\n"; /* * Run a Client Job -- Client already authorized @@ -116,6 +119,9 @@ void run_job(JCR *jcr) jcr->start_time = time(NULL); jcr->run_time = jcr->start_time; jcr->sendJobStatus(JS_Running); + + /* TODO: Remove when the new match_all is well tested */ + jcr->use_new_match_all = use_new_match_all; /* * A migrate or copy job does both a restore (read_data) and * a backup (append_data). @@ -135,7 +141,16 @@ void run_job(JCR *jcr) append_end_session(jcr); } else if (jcr->is_JobType(JT_MIGRATE) || jcr->is_JobType(JT_COPY)) { jcr->session_opened = true; - read_data_cmd(jcr); + /* send "3000 OK data" now to avoid a dead lock, the other side is also + * waiting for one. The old peace of code was reading the "3000 OK" reply + * at the end of the backup (not really appropriate). + * dedup need duplex communication with the other side and need the + * "3000 OK" to be out of the socket, and be handle here by the right + * peace of code */ + Dmsg0(215, "send OK_data\n"); + jcr->file_bsock->fsend(OK_data); + jcr->is_ok_data_sent = true; + Dmsg1(050, "Do: read_data_cmd file_bsock=%p\n", jcr->file_bsock); Dmsg0(050, "Do: receive for 3000 OK data then read\n"); if (!response(jcr, jcr->file_bsock, "3000 OK data\n", "Data received")) { Dmsg1(050, "Expect 3000 OK data, got: %s", jcr->file_bsock->msg); @@ -143,6 +158,7 @@ void run_job(JCR *jcr) jcr->file_bsock->signal(BNET_EOD); goto bail_out; } + read_data_cmd(jcr); jcr->file_bsock->signal(BNET_EOD); } else { /* Either a Backup or Restore job */ @@ -156,9 +172,11 @@ bail_out: jcr->setJobStatus(JS_Terminated); generate_daemon_event(jcr, "JobEnd"); generate_plugin_event(jcr, bsdEventJobEnd); + bash_spaces(jcr->StatusErrMsg); dir->fsend(Job_end, jcr->Job, jcr->JobStatus, jcr->JobFiles, - edit_uint64(jcr->JobBytes, ec1), jcr->JobErrors); + edit_uint64(jcr->JobBytes, ec1), jcr->JobErrors, jcr->StatusErrMsg); Dmsg1(100, "==== %s", dir->msg); + unbash_spaces(jcr->StatusErrMsg); dir->signal(BNET_EOD); /* send EOD to Director daemon */ free_plugins(jcr); /* release instantiated plugins */ garbage_collect_memory_pool(); @@ -195,11 +213,14 @@ void do_client_commands(JCR *jcr) if (!fd_cmds[i].func(jcr)) { /* do command */ /* Note fd->msg command may be destroyed by comm activity */ if (!job_canceled(jcr)) { + strip_trailing_junk(fd->msg); if (jcr->errmsg[0]) { - Jmsg1(jcr, M_FATAL, 0, _("Command error with FD, hanging up. ERR=%s\n"), - jcr->errmsg); + strip_trailing_junk(jcr->errmsg); + Jmsg2(jcr, M_FATAL, 0, _("Command error with FD msg=\"%s\", SD hanging up. ERR=%s\n"), + fd->msg, jcr->errmsg); } else { - Jmsg0(jcr, M_FATAL, 0, _("Command error with FD, hanging up.\n")); + Jmsg1(jcr, M_FATAL, 0, _("Command error with FD msg=\"%s\", SD hanging up.\n"), + fd->msg); } jcr->setJobStatus(JS_ErrorTerminated); } @@ -251,7 +272,7 @@ static bool append_end_session(JCR *jcr) { BSOCK *fd = jcr->file_bsock; - Dmsg1(120, ">filed: %s", fd->msg); + Dmsg1(120, "storemsg); if (!jcr->session_opened) { pm_strcpy(jcr->errmsg, _("Attempt to close non-open session.\n")); fd->fsend(NOT_opened); @@ -260,6 +281,43 @@ static bool append_end_session(JCR *jcr) return fd->fsend(OK_end); } +/* + * Test the FD/SD connectivity + */ +static bool sd_testnetwork_cmd(JCR *jcr) +{ + BSOCK *fd = jcr->file_bsock; + int64_t nb=0; + bool can_compress, ok=true; + + if (sscanf(fd->msg, "testnetwork bytes=%lld", &nb) != 1) { + return false; + } + /* We disable the comline compression for this test */ + can_compress = fd->can_compress(); + fd->clear_compress(); + + /* First, get data from the FD */ + while (fd->recv() > 0) { } + + /* Then, send back data to the FD */ + memset(fd->msg, 0xBB, sizeof_pool_memory(fd->msg)); + fd->msglen = sizeof_pool_memory(fd->msg); + + while(nb > 0 && ok) { + if (nb < fd->msglen) { + fd->msglen = nb; + } + ok = fd->send(); + nb -= fd->msglen; + } + fd->signal(BNET_EOD); + + if (can_compress) { + fd->set_compress(); + } + return true; +} /* * Append Open session command @@ -305,6 +363,7 @@ static bool append_close_session(JCR *jcr) Dmsg1(120, ">filed: %s", fd->msg); fd->signal(BNET_EOD); /* send EOD to File daemon */ + jcr->session_opened = false; return true; } @@ -373,6 +432,19 @@ static bool read_open_session(JCR *jcr) return true; } +static bool read_control_cmd(JCR *jcr) +{ + BSOCK *fd = jcr->file_bsock; + + Dmsg1(120, "Read control: %s\n", fd->msg); + if (!jcr->session_opened) { + fd->fsend(NOT_opened); + return false; + } + jcr->interactive_session = true; + return true; +} + /* * Read Close session command * Close the read session diff --git a/bacula/src/stored/fifo_dev.c b/bacula/src/stored/fifo_dev.c new file mode 100644 index 0000000000..6143a52168 --- /dev/null +++ b/bacula/src/stored/fifo_dev.c @@ -0,0 +1,51 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * fifo_dev.c -- low level operations on fifo devices + * + * written by, Kern Sibbald, MM + * separated from file_dev.c January 2016f4 + * + */ + +#include "bacula.h" +#include "stored.h" + +bool fifo_dev::open_device(DCR *dcr, int omode) +{ + return tape_dev::open_device(dcr, omode); +} + +boffset_t fifo_dev::lseek(DCR *dcr, boffset_t offset, int whence) +{ + /* Cannot seek */ + return 0; +} + +bool fifo_dev::truncate(DCR *dcr) +{ + /* Cannot truncate */ + return true; +} + +const char *fifo_dev::print_type() +{ + return "FIFO"; +} diff --git a/bacula/src/stored/fifo_dev.h b/bacula/src/stored/fifo_dev.h new file mode 100644 index 0000000000..72c6beac61 --- /dev/null +++ b/bacula/src/stored/fifo_dev.h @@ -0,0 +1,39 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Modified version of tape_dev.h + */ + +#ifndef __FIFO_DEV_ +#define __FIFO_DEV_ + +class fifo_dev : public tape_dev { +public: + + fifo_dev() { }; + ~fifo_dev() { }; + + /* DEVICE virtual functions that we redefine with our fifo code */ + bool open_device(DCR *dcr, int omode); + boffset_t lseek(DCR *dcr, boffset_t offset, int whence); + bool truncate(DCR *dcr); + const char *print_type(); +}; + +#endif /* __FIFO_DEV_ */ diff --git a/bacula/src/stored/file_dev.c b/bacula/src/stored/file_dev.c index 9dfc22698d..516274ef23 100644 --- a/bacula/src/stored/file_dev.c +++ b/bacula/src/stored/file_dev.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -28,6 +28,8 @@ #include "bacula.h" #include "stored.h" +static const int dbglvl = 100; + /* Imported functions */ const char *mode_to_str(int mode); @@ -35,7 +37,7 @@ const char *mode_to_str(int mode); /* default primitives are designed for file */ int DEVICE::d_open(const char *pathname, int flags) { - return ::open(pathname, flags); + return ::open(pathname, flags | O_CLOEXEC); } int DEVICE::d_close(int fd) @@ -64,13 +66,16 @@ ssize_t DEVICE::d_write(int fd, const void *buffer, size_t count) /* Rewind file device */ bool DEVICE::rewind(DCR *dcr) + { + Enter(dbglvl); Dmsg3(400, "rewind res=%d fd=%d %s\n", num_reserved(), m_fd, print_name()); state &= ~(ST_EOT|ST_EOF|ST_WEOT); /* remove EOF/EOT flags */ block_num = file = 0; file_size = 0; file_addr = 0; if (m_fd < 0) { + Mmsg1(errmsg, _("Rewind failed: device %s is not open.\n"), print_name()); return false; } if (is_file()) { @@ -90,7 +95,7 @@ bool DEVICE::rewind(DCR *dcr) * Returns: false on failure * true on success */ -bool DEVICE::reposition(DCR *dcr, uint32_t rfile, uint32_t rblock) +bool DEVICE::reposition(DCR *dcr, uint64_t raddr) { if (!is_open()) { dev_errno = EBADF; @@ -99,18 +104,15 @@ bool DEVICE::reposition(DCR *dcr, uint32_t rfile, uint32_t rblock) return false; } - boffset_t pos = (((boffset_t)rfile)<<32) | rblock; - Dmsg1(100, "===== lseek to %d\n", (int)pos); - if (lseek(dcr, pos, SEEK_SET) == (boffset_t)-1) { + Dmsg1(100, "===== lseek to %llu\n", raddr); + if (lseek(dcr, (boffset_t)raddr, SEEK_SET) == (boffset_t)-1) { berrno be; dev_errno = errno; Mmsg2(errmsg, _("lseek error on %s. ERR=%s.\n"), print_name(), be.bstrerror()); return false; } - file = rfile; - block_num = rblock; - file_addr = pos; + file_addr = raddr; return true; } @@ -118,29 +120,32 @@ bool DEVICE::reposition(DCR *dcr, uint32_t rfile, uint32_t rblock) /* Seek to specified place */ boffset_t DEVICE::lseek(DCR *dcr, boffset_t offset, int whence) { - switch (dev_type) { - case B_FILE_DEV: #if defined(HAVE_WIN32) - return ::_lseeki64(m_fd, (__int64)offset, whence); + return ::_lseeki64(m_fd, (__int64)offset, whence); #else - return ::lseek(m_fd, offset, whence); + return ::lseek(m_fd, offset, whence); #endif - } - return -1; } /* - * Open a file device. + * Open a file device. For Aligned type we open both Volumes */ -void DEVICE::open_file_device(DCR *dcr, int omode) +bool file_dev::open_device(DCR *dcr, int omode) { POOL_MEM archive_name(PM_FNAME); struct stat sp; + Enter(dbglvl); + if (DEVICE::open_device(dcr, omode)) { + Leave(dbglvl); + return true; + } + omode = openmode; + get_autochanger_loaded_slot(dcr); /* - * Handle opening of File Archive (not a tape) + * Handle opening of File Autochanger */ pm_strcpy(archive_name, dev_name); @@ -158,7 +163,8 @@ void DEVICE::open_file_device(DCR *dcr, int omode) pm_strcpy(dcr->jcr->errmsg, errmsg); } clear_opened(); - return; + Leave(dbglvl); + return false; } /* If not /dev/null concatenate VolumeName */ @@ -172,20 +178,21 @@ void DEVICE::open_file_device(DCR *dcr, int omode) mount(1); /* do mount if required */ - openmode = omode; set_mode(omode); /* If creating file, give 0640 permissions */ Dmsg3(100, "open disk: mode=%s open(%s, 0x%x, 0640)\n", mode_to_str(omode), archive_name.c_str(), mode); /* Use system open() */ - if ((m_fd = ::open(archive_name.c_str(), mode, 0640)) < 0) { + if ((m_fd = ::open(archive_name.c_str(), mode|O_CLOEXEC, 0640)) < 0) { berrno be; dev_errno = errno; Mmsg3(errmsg, _("Could not open(%s,%s,0640): ERR=%s\n"), archive_name.c_str(), mode_to_str(omode), be.bstrerror()); Dmsg1(40, "open failed: %s", errmsg); } else { + /* Open is OK, now let device get control */ Dmsg2(40, "Did open(%s,%s,0640)\n", archive_name.c_str(), mode_to_str(omode)); + device_specific_open(dcr); } if (m_fd >= 0) { dev_errno = 0; @@ -201,14 +208,18 @@ void DEVICE::open_file_device(DCR *dcr, int omode) pm_strcpy(dcr->jcr->errmsg, errmsg); } } - Dmsg1(100, "open dev: disk fd=%d opened\n", m_fd); -} + Dmsg1(100, "open dev: disk fd=%d opened, aligned=%d\n", m_fd); + state |= preserve; /* reset any important state info */ + Leave(dbglvl); + return m_fd >= 0; +} /* - * Truncate a volume. + * Truncate a volume. If this is aligned disk, we + * truncate both volumes. */ -bool DEVICE::truncate(DCR *dcr) /* We need the DCR for DVD-writing */ +bool DEVICE::truncate(DCR *dcr) { struct stat st; DEVICE *dev = this; @@ -220,75 +231,76 @@ bool DEVICE::truncate(DCR *dcr) /* We need the DCR for DVD-writing */ case B_TAPE_DEV: /* maybe we should rewind and write and eof ???? */ return true; /* we don't really truncate tapes */ - case B_FILE_DEV: - /* Do truncate for 1 or 2 devices */ - for ( ;; ) { - Dmsg1(100, "Truncate fd=%d\n", dev->m_fd); - if (ftruncate(dev->m_fd, 0) != 0) { - berrno be; - Mmsg2(errmsg, _("Unable to truncate device %s. ERR=%s\n"), - print_name(), be.bstrerror()); - return false; - } + default: + break; + } - /* - * Check for a successful ftruncate() and issue a work-around for devices - * (mostly cheap NAS) that don't support truncation. - * Workaround supplied by Martin Schmid as a solution to bug #1011. - * 1. close file - * 2. delete file - * 3. open new file with same mode - * 4. change ownership to original - */ - - if (fstat(dev->m_fd, &st) != 0) { - berrno be; - Mmsg2(errmsg, _("Unable to stat device %s. ERR=%s\n"), - print_name(), be.bstrerror()); - return false; - } + /* Do truncate for 1 or 2 devices */ + Dmsg2(100, "Truncate adata=%d fd=%d\n", dev->adata, dev->m_fd); + if (ftruncate(dev->m_fd, 0) != 0) { + berrno be; + Mmsg2(errmsg, _("Unable to truncate device %s. ERR=%s\n"), + print_name(), be.bstrerror()); + return false; + } - if (st.st_size != 0) { /* ftruncate() didn't work */ - POOL_MEM archive_name(PM_FNAME); - - pm_strcpy(archive_name, dev_name); - if (!IsPathSeparator(archive_name.c_str()[strlen(archive_name.c_str())-1])) { - pm_strcat(archive_name, "/"); - } - pm_strcat(archive_name, dcr->VolumeName); - - Mmsg2(errmsg, _("Device %s doesn't support ftruncate(). Recreating file %s.\n"), - print_name(), archive_name.c_str()); - - /* Close file and blow it away */ - ::close(dev->m_fd); - ::unlink(archive_name.c_str()); - - /* Recreate the file -- of course, empty */ - dev->set_mode(CREATE_READ_WRITE); - if ((dev->m_fd = ::open(archive_name.c_str(), mode, st.st_mode)) < 0) { - berrno be; - dev_errno = errno; - Mmsg2(errmsg, _("Could not reopen: %s, ERR=%s\n"), archive_name.c_str(), - be.bstrerror()); - Dmsg1(40, "reopen failed: %s", errmsg); - Emsg0(M_FATAL, 0, errmsg); - return false; - } - - /* Reset proper owner */ - chown(archive_name.c_str(), st.st_uid, st.st_gid); - } - break; + /* + * Check for a successful ftruncate() and issue a work-around for devices + * (mostly cheap NAS) that don't support truncation. + * Workaround supplied by Martin Schmid as a solution to bug #1011. + * 1. close file + * 2. delete file + * 3. open new file with same mode + * 4. change ownership to original + */ + + if (fstat(dev->m_fd, &st) != 0) { + berrno be; + Mmsg2(errmsg, _("Unable to stat device %s. ERR=%s\n"), + print_name(), be.bstrerror()); + return false; + } + + if (st.st_size != 0) { /* ftruncate() didn't work */ + POOL_MEM archive_name(PM_FNAME); + + pm_strcpy(archive_name, dev_name); + if (!IsPathSeparator(archive_name.c_str()[strlen(archive_name.c_str())-1])) { + pm_strcat(archive_name, "/"); } - return true; + pm_strcat(archive_name, dcr->VolumeName); + if (dev->is_adata()) { + pm_strcat(archive_name, ADATA_EXTENSION); + } + + Mmsg2(errmsg, _("Device %s doesn't support ftruncate(). Recreating file %s.\n"), + print_name(), archive_name.c_str()); + + /* Close file and blow it away */ + ::close(dev->m_fd); + ::unlink(archive_name.c_str()); + + /* Recreate the file -- of course, empty */ + dev->set_mode(CREATE_READ_WRITE); + if ((dev->m_fd = ::open(archive_name.c_str(), mode|O_CLOEXEC, st.st_mode)) < 0) { + berrno be; + dev_errno = errno; + Mmsg2(errmsg, _("Could not reopen: %s, ERR=%s\n"), archive_name.c_str(), + be.bstrerror()); + Dmsg1(40, "reopen failed: %s", errmsg); + Emsg0(M_FATAL, 0, errmsg); + return false; + } + + /* Reset proper owner */ + chown(archive_name.c_str(), st.st_uid, st.st_gid); } - return false; + return true; } /* - * (Un)mount the device (either a FILE or DVD device) + * (Un)mount the device */ bool DEVICE::mount_file(int mount, int dotimeout) { @@ -410,3 +422,123 @@ get_out: Dmsg1(200, "============ mount=%d\n", mount); return true; } + +/* + * Check if the current position on the volume corresponds to + * what is in the catalog. + * + */ +bool file_dev::is_eod_valid(DCR *dcr) +{ + JCR *jcr = dcr->jcr; + + if (has_cap(CAP_LSEEK)) { + char ed1[50], ed2[50]; + boffset_t ameta_size, adata_size, size; + + ameta_size = lseek(dcr, (boffset_t)0, SEEK_END); + adata_size = get_adata_size(dcr); + size = ameta_size + adata_size; + if (VolCatInfo.VolCatAmetaBytes == (uint64_t)ameta_size && + VolCatInfo.VolCatAdataBytes == (uint64_t)adata_size) { + if (is_aligned()) { + Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volumes \"%s\"" + " ameta size=%s adata size=%s\n"), dcr->VolumeName, + edit_uint64_with_commas(VolCatInfo.VolCatAmetaBytes, ed1), + edit_uint64_with_commas(VolCatInfo.VolCatAdataBytes, ed2)); + } else { + Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volume \"%s\"" + " size=%s\n"), dcr->VolumeName, + edit_uint64_with_commas(VolCatInfo.VolCatAmetaBytes, ed1)); + } + } else if ((uint64_t)ameta_size >= VolCatInfo.VolCatAmetaBytes && + (uint64_t)adata_size >= VolCatInfo.VolCatAdataBytes) { + if ((uint64_t)ameta_size != VolCatInfo.VolCatAmetaBytes) { + Jmsg(jcr, M_WARNING, 0, _("For Volume \"%s\":\n" + " The sizes do not match! Metadata Volume=%s Catalog=%s\n" + " Correcting Catalog\n"), + dcr->VolumeName, edit_uint64_with_commas(ameta_size, ed1), + edit_uint64_with_commas(VolCatInfo.VolCatAmetaBytes, ed2)); + } + if ((uint64_t)adata_size != VolCatInfo.VolCatAdataBytes) { + Jmsg(jcr, M_WARNING, 0, _("For aligned Volume \"%s\":\n" + " Aligned sizes do not match! Aligned Volume=%s Catalog=%s\n" + " Correcting Catalog\n"), + dcr->VolumeName, edit_uint64_with_commas(adata_size, ed1), + edit_uint64_with_commas(VolCatInfo.VolCatAdataBytes, ed2)); + } + VolCatInfo.VolCatAmetaBytes = ameta_size; + VolCatInfo.VolCatAdataBytes = adata_size; + VolCatInfo.VolCatBytes = size; + VolCatInfo.VolCatFiles = (uint32_t)(size >> 32); + if (!dir_update_volume_info(dcr, false, true)) { + Jmsg(jcr, M_WARNING, 0, _("Error updating Catalog\n")); + dcr->mark_volume_in_error(); + return false; + } + } else { + Mmsg(jcr->errmsg, _("Bacula cannot write on disk Volume \"%s\" because: " + "The sizes do not match! Volume=%s Catalog=%s\n"), + dcr->VolumeName, + edit_uint64_with_commas(size, ed1), + edit_uint64_with_commas(VolCatInfo.VolCatBytes, ed2)); + Jmsg(jcr, M_ERROR, 0, jcr->errmsg); + Dmsg0(100, jcr->errmsg); + dcr->mark_volume_in_error(); + return false; + } + } + return true; +} + + +/* + * Position device to end of medium (end of data) + * Returns: true on succes + * false on error + */ +bool file_dev::eod(DCR *dcr) +{ + boffset_t pos; + + Enter(100); + if (m_fd < 0) { + dev_errno = EBADF; + Mmsg1(errmsg, _("Bad call to eod. Device %s not open\n"), print_name()); + Dmsg1(100, "%s", errmsg); + return false; + } + + if (at_eot()) { + Leave(100); + return true; + } + clear_eof(); /* remove EOF flag */ + block_num = file = 0; + file_size = 0; + file_addr = 0; + if (is_fifo()) { + Leave(100); + return true; + } + pos = lseek(dcr, (boffset_t)0, SEEK_END); + Dmsg1(200, "====== Seek to %lld\n", pos); + if (pos >= 0) { + update_pos(dcr); + set_eot(); + Leave(100); + return true; + } + dev_errno = errno; + berrno be; + Mmsg2(errmsg, _("lseek error on %s. ERR=%s.\n"), + print_name(), be.bstrerror()); + Dmsg1(100, "%s", errmsg); + Leave(100); + return false; +} + +const char *file_dev::print_type() +{ + return "File"; +} diff --git a/bacula/src/stored/file_dev.h b/bacula/src/stored/file_dev.h index 6f765601a6..0c841cf26f 100644 --- a/bacula/src/stored/file_dev.h +++ b/bacula/src/stored/file_dev.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -28,7 +28,10 @@ public: file_dev() { }; ~file_dev() { m_fd = -1; }; -// bool mount_file(int mount, int dotimeout); + bool is_eod_valid(DCR *dcr); + bool eod(DCR *dcr); + bool open_device(DCR *dcr, int omode); + const char *print_type(); }; #endif /* __FILE_DEV_ */ diff --git a/bacula/src/stored/file_driver.c b/bacula/src/stored/file_driver.c new file mode 100644 index 0000000000..65d5472760 --- /dev/null +++ b/bacula/src/stored/file_driver.c @@ -0,0 +1,31 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Routines for writing to a file from the Cloud device. + * + * This is for testing purposes only. + # + * NOTE!!! This cloud driver is not compatible with + * any disk-changer script for changing Volumes. + * It does however work with Bacula Virtual autochangers. + * + * Written by Kern Sibbald, May MMXVI + * + */ + diff --git a/bacula/src/stored/file_driver.h b/bacula/src/stored/file_driver.h new file mode 100644 index 0000000000..b8f757fd96 --- /dev/null +++ b/bacula/src/stored/file_driver.h @@ -0,0 +1,40 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Routines for writing to the Cloud using S3 protocol. + * + * Written by Kern Sibbald, May MMXVI + */ + +#ifndef _FILE_DRV_H +#define _FILE_DRV_H + +#include "bacula.h" +#include "stored.h" +#include "cloud_driver.h" /* get base class definitions */ + +class file_driver: public cloud_driver { +public: + file_driver() { + }; + ~file_driver() { + }; +}; + +#endif /* _FILE_DRV_H */ diff --git a/bacula/src/stored/global.c b/bacula/src/stored/global.c new file mode 100644 index 0000000000..13868618bb --- /dev/null +++ b/bacula/src/stored/global.c @@ -0,0 +1,26 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "bacula.h" +#include "stored.h" + +STORES *me = NULL; /* our Global resource */ +bool forge_on = false; /* proceed inspite of I/O errors */ +pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; diff --git a/bacula/src/stored/hello.c b/bacula/src/stored/hello.c index 83e9d154e9..49c3130825 100644 --- a/bacula/src/stored/hello.c +++ b/bacula/src/stored/hello.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -36,22 +36,26 @@ const int dbglvl = 50; /* * SD_VERSION history + * Note: Enterprise versions now numbered in 30000 + * and community is at SD version 3 * None prior to 06Aug13 * 1 - Skipped - * 2 - Skipped + * 2 - Skipped * 3 22Feb14 - Added SD->SD with SD_Calls_Client - * 4 - Skipped + * 4 22Jun14 - Skipped * 305 04Jun15 - Added JobMedia queueing + * 306 20Mar15 - Added comm line compression */ -#define SD_VERSION 305 /* Community SD version */ -#define FD_VERSION 305 /* Community FD version */ +#define SD_VERSION 306 /* Community SD version */ +#define FD_VERSION 214 /* Community FD version */ static char hello_sd[] = "Hello Bacula SD: Start Job %s %d %d\n"; static char Sorry[] = "3999 No go\n"; static char OK_hello[] = "3000 OK Hello %d\n"; + /********************************************************************* * * Validate hello from the Director. @@ -89,6 +93,12 @@ bool validate_dir_hello(JCR* jcr) return false; } + if (dir_version >= 1 && me->comm_compression) { + dir->set_compress(); + } else { + dir->clear_compress(); + Dmsg0(050, "**** No SD compression to Dir\n"); + } director = NULL; unbash_spaces(dirname); foreach_res(director, R_DIRECTOR) { @@ -164,6 +174,14 @@ void handle_client_connection(BSOCK *fd) fd->set_jcr(jcr); Dmsg2(050, "fd_version=%d sd_version=%d\n", fd_version, sd_version); + /* Turn on compression for newer FDs */ + if (fd_version >= 214 || sd_version >= 306) { + fd->set_compress(); /* set compression allowed */ + } else { + fd->clear_compress(); + Dmsg0(050, "*** No SD compression to FD\n"); + } + /* * Authenticate the Client (FD or SD) */ @@ -263,6 +281,13 @@ bool read_client_hello(JCR *jcr) jcr->FDVersion = fd_version; jcr->SDVersion = sd_version; Dmsg1(050, "FDVersion=%d\n", fd_version); + /* Turn on compression for newer FDs, except for Community version */ + if (jcr->FDVersion >= 9 && jcr->FDVersion != 213 && me->comm_compression) { + cl->set_compress(); /* set compression allowed */ + } else { + cl->clear_compress(); + Dmsg0(050, "*** No SD compression to FD\n"); + } return true; } @@ -295,7 +320,6 @@ bool send_hello_sd(JCR *jcr, char *Job) if (!rtn) { return false; } - return true; } diff --git a/bacula/src/stored/init_dev.c b/bacula/src/stored/init_dev.c new file mode 100644 index 0000000000..01d6eb6f5f --- /dev/null +++ b/bacula/src/stored/init_dev.c @@ -0,0 +1,474 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Initialize a single device. + * + * This code was split from dev.c in January 2016 + * + * written by, Kern Sibbald, MM + */ + +#include "bacula.h" +#include "stored.h" +#include + +/* Define possible extensions */ +#if defined(HAVE_WIN32) +#define DRV_EXT ".dll" +#elif defined(HAVE_DARWIN_OS) +#define DRV_EXT ".dylib" +#else +#define DRV_EXT ".so" +#endif + +#ifndef RTLD_NOW +#define RTLD_NOW 2 +#endif + +/* Forward referenced functions */ +extern "C" { +typedef DEVICE *(*newDriver_t)(JCR *jcr, DEVRES *device); +} + +static DEVICE *load_driver(JCR *jcr, DEVRES *device); + +/* + * Driver item for driver table +*/ +struct driver_item { + const char *name; + void *handle; + newDriver_t newDriver; + bool builtin; + bool loaded; +}; + +/* + * Driver table. Must be in same order as the B_xxx_DEV type + * name handle, builtin loaded + */ +static driver_item driver_tab[] = { +/* name handle, newDriver builtin loaded */ + {"file", NULL, NULL, true, true}, + {"tape", NULL, NULL, true, true}, + {"none", NULL, NULL, true, true}, /* deprecated was DVD */ + {"fifo", NULL, NULL, true, true}, + {"vtape", NULL, NULL, true, true}, + {"ftp", NULL, NULL, true, true}, + {"vtl", NULL, NULL, true, true}, + {"none", NULL, NULL, true, true}, /* B_ADATA_DEV */ + {"aligned", NULL, NULL, false, false}, + {"none", NULL, NULL, true, true}, /* deprecated was old dedup */ + {"null", NULL, NULL, true, true}, + {"none", NULL, NULL, true, true}, /* deprecated B_VALIGNED_DEV */ + {"none", NULL, NULL, true, true}, /* deprecated B_VDEDUP_DEV */ + {"cloud", NULL, NULL, false, false}, + {"none", NULL, NULL, false, false}, + {NULL, NULL, NULL, false, false} +}; + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + +/* The alist should be created with not_owned_by_alist argument */ +void sd_list_loaded_drivers(alist *list) +{ + for(int i=0 ; driver_tab[i].name != NULL; i++) { + if (driver_tab[i].loaded && !driver_tab[i].builtin) { + list->append((void*)driver_tab[i].name); + } + } +} + +/* + * Allocate and initialize the DEVICE structure + * Note, if dev is non-NULL, it is already allocated, + * thus we neither allocate it nor free it. This allows + * the caller to put the packet in shared memory. + * + * Note, for a tape, the device->device_name is the device name + * (e.g. /dev/nst0), and for a file, the device name + * is the directory in which the file will be placed. + * + */ +DEVICE *init_dev(JCR *jcr, DEVRES *device, bool adata) +{ + struct stat statp; + DEVICE *dev = NULL; + uint32_t n_drivers; + + generate_global_plugin_event(bsdGlobalEventDeviceInit, device); + Dmsg1(150, "init_dev dev_type=%d\n", device->dev_type); + /* If no device type specified, try to guess */ + if (!device->dev_type) { + /* Check that device is available */ + if (stat(device->device_name, &statp) < 0) { + berrno be; + Jmsg3(jcr, M_ERROR, 0, _("[SE0001] Unable to stat device %s at %s: ERR=%s\n"), + device->hdr.name, device->device_name, be.bstrerror()); + return NULL; + } + if (S_ISDIR(statp.st_mode)) { + device->dev_type = B_FILE_DEV; + } else if (S_ISCHR(statp.st_mode)) { + device->dev_type = B_TAPE_DEV; + } else if (S_ISFIFO(statp.st_mode)) { + device->dev_type = B_FIFO_DEV; +#ifdef USE_VTAPE + /* must set DeviceType = Vtape + * in normal mode, autodetection is disabled + */ + } else if (S_ISREG(statp.st_mode)) { + device->dev_type = B_VTAPE_DEV; +#endif + } else if (!(device->cap_bits & CAP_REQMOUNT)) { + Jmsg2(jcr, M_ERROR, 0, _("[SE0002] %s is an unknown device type. Must be tape or directory." + " st_mode=%x\n"), + device->device_name, statp.st_mode); + return NULL; + } + if (strcmp(device->device_name, "/dev/null") == 0) { + device->dev_type = B_NULL_DEV; + } + } + + /* Count drivers */ + for (n_drivers=0; driver_tab[n_drivers].name; n_drivers++) { }; + Dmsg1(100, "Num drivers=%d\n", n_drivers); + + /* If invalid dev_type get out */ + if (device->dev_type < 0 || device->dev_type > n_drivers) { + Jmsg2(jcr, M_FATAL, 0, _("[SF0001] Invalid device type=%d name=\"%s\"\n"), + device->dev_type, device->hdr.name); + return NULL; + } + Dmsg5(100, "loadable=%d type=%d loaded=%d name=%s handle=%p\n", + !driver_tab[device->dev_type-1].builtin, + device->dev_type, + driver_tab[device->dev_type-1].loaded, + driver_tab[device->dev_type-1].name, + driver_tab[device->dev_type-1].handle); + if (driver_tab[device->dev_type-1].builtin) { + /* Built-in driver */ + switch (device->dev_type) { +#ifdef HAVE_WIN32 + case B_TAPE_DEV: +// dev = New(win_tape_dev); + break; + case B_ADATA_DEV: + case B_ALIGNED_DEV: + case B_FILE_DEV: + dev = New(win_file_dev); + dev->capabilities |= CAP_LSEEK; + break; + case B_NULL_DEV: + dev = New(win_file_dev); + break; +#else + case B_VTAPE_DEV: + dev = New(vtape); + break; + case B_TAPE_DEV: + dev = New(tape_dev); + break; + case B_FILE_DEV: + dev = New(file_dev); + dev->capabilities |= CAP_LSEEK; + break; + case B_NULL_DEV: + dev = New(null_dev); + break; + case B_FIFO_DEV: + dev = New(fifo_dev); + break; +#endif + default: + Jmsg2(jcr, M_FATAL, 0, _("[SF0002] Unknown device type=%d device=\"%s\"\n"), + device->dev_type, device->hdr.name); + return NULL; + } + } else { + /* Loadable driver */ + dev = load_driver(jcr, device); + } + if (!dev) { + return NULL; + } + + dev->adata = adata; + + /* Keep the device ID in the DEVICE struct to identify the hardware */ + if (dev->is_file() && stat(dev->archive_name(), &statp) == 0) { + dev->devno = statp.st_dev; + } + + dev->device_generic_init(jcr, device); + + /* Do device specific initialization */ + dev->device_specific_init(jcr, device); + + /* ***FIXME*** move to fifo driver */ + if (dev->is_fifo()) { + dev->capabilities |= CAP_STREAM; /* set stream device */ + } + + return dev; +} + +/* + * Do all the generic initialization here. Same for all devices. + */ +void DEVICE::device_generic_init(JCR *jcr, DEVRES *device) +{ + struct stat statp; + DEVICE *dev = this; + DCR *dcr = NULL; + int errstat; + uint32_t max_bs; + + dev->clear_slot(); /* unknown */ + + /* Copy user supplied device parameters from Resource */ + dev->dev_name = get_memory(strlen(device->device_name)+1); + pm_strcpy(dev->dev_name, device->device_name); + dev->prt_name = get_memory(strlen(device->device_name) + strlen(device->hdr.name) + 20); + /* We edit "Resource-name" (physical-name) */ + Mmsg(dev->prt_name, "\"%s\" (%s)", device->hdr.name, device->device_name); + Dmsg1(400, "Allocate dev=%s\n", dev->print_name()); + dev->capabilities = device->cap_bits; + dev->min_free_space = device->min_free_space; + dev->min_block_size = device->min_block_size; + dev->max_block_size = device->max_block_size; + dev->max_volume_size = device->max_volume_size; + dev->max_file_size = device->max_file_size; + dev->padding_size = device->padding_size; + dev->file_alignment = device->file_alignment; + dev->max_concurrent_jobs = device->max_concurrent_jobs; + dev->volume_capacity = device->volume_capacity; + dev->max_rewind_wait = device->max_rewind_wait; + dev->max_open_wait = device->max_open_wait; + dev->vol_poll_interval = device->vol_poll_interval; + dev->max_spool_size = device->max_spool_size; + dev->drive_index = device->drive_index; + dev->enabled = device->enabled; + dev->autoselect = device->autoselect; + dev->read_only = device->read_only; + dev->dev_type = device->dev_type; + dev->device = device; + if (dev->is_tape()) { /* No parts on tapes */ + dev->max_part_size = 0; + } else { + dev->max_part_size = device->max_part_size; + } + /* Sanity check */ + if (dev->vol_poll_interval && dev->vol_poll_interval < 60) { + dev->vol_poll_interval = 60; + } + + if (!device->dev) { + device->dev = dev; + } + + /* If the device requires mount : + * - Check that the mount point is available + * - Check that (un)mount commands are defined + */ + if (dev->is_file() && dev->requires_mount()) { + if (!device->mount_point || stat(device->mount_point, &statp) < 0) { + berrno be; + dev->dev_errno = errno; + Jmsg2(jcr, M_ERROR_TERM, 0, _("[SA0003] Unable to stat mount point %s: ERR=%s\n"), + device->mount_point, be.bstrerror()); + } + + if (!device->mount_command || !device->unmount_command) { + Jmsg0(jcr, M_ERROR_TERM, 0, _("[SA0004] Mount and unmount commands must defined for a device which requires mount.\n")); + } + } + + + /* Sanity check */ + if (dev->max_block_size == 0) { + max_bs = DEFAULT_BLOCK_SIZE; + } else { + max_bs = dev->max_block_size; + } + if (dev->min_block_size > max_bs) { + Jmsg(jcr, M_ERROR_TERM, 0, _("[SA0005] Min block size > max on device %s\n"), + dev->print_name()); + } + if (dev->max_block_size > 4096000) { + Jmsg3(jcr, M_ERROR, 0, _("[SA0006] Block size %u on device %s is too large, using default %u\n"), + dev->max_block_size, dev->print_name(), DEFAULT_BLOCK_SIZE); + dev->max_block_size = 0; + } + if (dev->max_block_size % TAPE_BSIZE != 0) { + Jmsg3(jcr, M_WARNING, 0, _("[SW0007] Max block size %u not multiple of device %s block size=%d.\n"), + dev->max_block_size, dev->print_name(), TAPE_BSIZE); + } + if (dev->max_volume_size != 0 && dev->max_volume_size < (dev->max_block_size << 4)) { + Jmsg(jcr, M_ERROR_TERM, 0, _("[SA0008] Max Vol Size < 8 * Max Block Size for device %s\n"), + dev->print_name()); + } + + dev->errmsg = get_pool_memory(PM_EMSG); + *dev->errmsg = 0; + + if ((errstat = dev->init_mutex()) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0009] Unable to init mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = pthread_cond_init(&dev->wait, NULL)) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0010] Unable to init cond variable: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = pthread_cond_init(&dev->wait_next_vol, NULL)) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0011] Unable to init cond variable: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = pthread_mutex_init(&dev->spool_mutex, NULL)) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0012] Unable to init spool mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = dev->init_acquire_mutex()) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0013] Unable to init acquire mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = dev->init_freespace_mutex()) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0014] Unable to init freespace mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = dev->init_read_acquire_mutex()) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0015] Unable to init read acquire mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = dev->init_volcat_mutex()) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0016] Unable to init volcat mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + if ((errstat = dev->init_dcrs_mutex()) != 0) { + berrno be; + dev->dev_errno = errstat; + Mmsg1(dev->errmsg, _("[SA0017] Unable to init dcrs mutex: ERR=%s\n"), be.bstrerror(errstat)); + Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg); + } + + dev->set_mutex_priorities(); + + dev->clear_opened(); + dev->attached_dcrs = New(dlist(dcr, &dcr->dev_link)); + Dmsg2(100, "init_dev: tape=%d dev_name=%s\n", dev->is_tape(), dev->dev_name); + dev->initiated = true; +} + +static DEVICE *load_driver(JCR *jcr, DEVRES *device) +{ + POOL_MEM fname(PM_FNAME); + DEVICE *dev; + driver_item *drv; + const char *slash; + void *pHandle; + int len; + newDriver_t newDriver; + + P(mutex); + if (!me->plugin_directory) { + Jmsg2(jcr, M_FATAL, 0, _("[SF0018] Plugin directory not defined. Cannot load SD %s driver for device %s.\n"), + driver_tab[device->dev_type - 1], device->hdr.name); + V(mutex); + return NULL; + } + len = strlen(me->plugin_directory); + if (len == 0) { + Jmsg0(jcr, M_FATAL, 0, _("[SF0019] Plugin directory not defined. Cannot load drivers.\n")); + V(mutex); + return NULL; + } + + if (IsPathSeparator(me->plugin_directory[len - 1])) { + slash = ""; + } else { + slash = "/"; + } + + Dmsg5(100, "loadable=%d type=%d loaded=%d name=%s handle=%p\n", + !driver_tab[device->dev_type-1].builtin, + device->dev_type, + driver_tab[device->dev_type-1].loaded, + driver_tab[device->dev_type-1].name, + driver_tab[device->dev_type-1].handle); + drv = &driver_tab[device->dev_type - 1]; + Mmsg(fname, "%s%sbacula-sd-%s-driver%s%s", me->plugin_directory, slash, + drv->name, "-" VERSION, DRV_EXT); + if (!drv->loaded) { + Dmsg1(10, "Open SD driver at %s\n", fname.c_str()); + pHandle = dlopen(fname.c_str(), RTLD_NOW); + if (pHandle) { + Dmsg2(100, "Driver=%s handle=%p\n", drv->name, pHandle); + /* Get global entry point */ + Dmsg1(10, "Lookup \"BaculaSDdriver\" in driver=%s\n", drv->name); + newDriver = (newDriver_t)dlsym(pHandle, "BaculaSDdriver"); + Dmsg2(10, "Driver=%s entry point=%p\n", drv->name, newDriver); + if (!newDriver) { + const char *error = dlerror(); + Jmsg(NULL, M_ERROR, 0, _("[SE0003] Lookup of symbol \"BaculaSDdriver\" in driver %s for device %s failed: ERR=%s\n"), + device->hdr.name, fname.c_str(), NPRT(error)); + Dmsg2(10, "Lookup of symbol \"BaculaSDdriver\" driver=%s failed: ERR=%s\n", + fname.c_str(), NPRT(error)); + dlclose(pHandle); + V(mutex); + return NULL; + } + drv->handle = pHandle; + drv->loaded = true; + drv->newDriver = newDriver; + } else { + /* dlopen failed */ + const char *error = dlerror(); + Jmsg3(jcr, M_FATAL, 0, _("[SF0020] dlopen of SD driver=%s at %s failed: ERR=%s\n"), + drv->name, fname.c_str(), NPRT(error)); + Dmsg2(000, "dlopen plugin %s failed: ERR=%s\n", fname.c_str(), + NPRT(error)); + V(mutex); + return NULL; + } + } else { + Dmsg1(10, "SD driver=%s is already loaded.\n", drv->name); + } + + /* Call driver initialization */ + dev = drv->newDriver(jcr, device); + V(mutex); + return dev; +} diff --git a/bacula/src/stored/job.c b/bacula/src/stored/job.c index 721768544c..a89014c2ee 100644 --- a/bacula/src/stored/job.c +++ b/bacula/src/stored/job.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -326,6 +326,7 @@ void stored_free_jcr(JCR *jcr) delete jcr->jobmedia_queue; jcr->jobmedia_queue = NULL; } + if (jcr->dir_bsock) { Dmsg2(800, "Send terminate jid=%d %p\n", jcr->JobId, jcr); jcr->dir_bsock->signal(BNET_EOD); diff --git a/bacula/src/stored/label.c b/bacula/src/stored/label.c index 3317c7f3ca..f205dcd3cb 100644 --- a/bacula/src/stored/label.c +++ b/bacula/src/stored/label.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -27,11 +27,10 @@ #include "bacula.h" /* pull in global headers */ #include "stored.h" /* pull in Storage Deamon headers */ +static const int dbglvl = 100; + /* Forward referenced functions */ -static void create_volume_label_record(DCR *dcr, DEVICE *dev, DEV_RECORD *rec, bool alt); -static bool sub_write_volume_label_to_block(DCR *dcr); -static bool sub_write_new_volume_label_to_dev(DCR *dcr, const char *VolName, - const char *PoolName, bool relabel, bool dvdnow); +static void create_volume_label_record(DCR *dcr, DEVICE *dev, DEV_RECORD *rec, bool adata); /* * Read the volume label @@ -52,15 +51,16 @@ static bool sub_write_new_volume_label_to_dev(DCR *dcr, const char *VolName, * VOL_VERSION_ERROR label has wrong version * VOL_LABEL_ERROR bad label type * VOL_NO_MEDIA no media in drive + * VOL_TYPE_ERROR aligned/non-aligned/dedup error * * The dcr block is emptied on return, and the Volume is * rewound. * + * Handle both the ameta and adata volumes. */ -int read_dev_volume_label(DCR *dcr) +int DEVICE::read_dev_volume_label(DCR *dcr) { JCR *jcr = dcr->jcr; - DEVICE * volatile dev = dcr->dev; char *VolName = dcr->VolumeName; DEV_RECORD *record; bool ok = false; @@ -69,36 +69,36 @@ int read_dev_volume_label(DCR *dcr) bool want_ansi_label; bool have_ansi_label = false; - Enter(200); - Dmsg4(100, "Enter read_volume_label res=%d device=%s vol=%s dev_Vol=%s\n", - dev->num_reserved(), dev->print_name(), VolName, - dev->VolHdr.VolumeName[0]?dev->VolHdr.VolumeName:"*NULL*"); + Enter(dbglvl); + Dmsg5(dbglvl, "Enter read_volume_label adata=%d res=%d device=%s vol=%s dev_Vol=%s\n", + block->adata, num_reserved(), print_name(), VolName, + VolHdr.VolumeName[0]?VolHdr.VolumeName:"*NULL*"); - if (!dev->is_open()) { - if (!dev->open(dcr, OPEN_READ_ONLY)) { - Leave(200); + if (!is_open()) { + if (!open_device(dcr, OPEN_READ_ONLY)) { + Leave(dbglvl); return VOL_IO_ERROR; } } - dev->clear_labeled(); - dev->clear_append(); - dev->clear_read(); - dev->label_type = B_BACULA_LABEL; + clear_labeled(); + clear_append(); + clear_read(); + label_type = B_BACULA_LABEL; - if (!dev->rewind(dcr)) { + if (!rewind(dcr)) { Mmsg(jcr->errmsg, _("Couldn't rewind %s device %s: ERR=%s\n"), - dev->print_type(), dev->print_name(), dev->print_errmsg()); - Dmsg1(130, "return VOL_NO_MEDIA: %s", jcr->errmsg); - Leave(200); + print_type(), print_name(), print_errmsg()); + Dmsg1(dbglvl, "return VOL_NO_MEDIA: %s", jcr->errmsg); + Leave(dbglvl); return VOL_NO_MEDIA; } - bstrncpy(dev->VolHdr.Id, "**error**", sizeof(dev->VolHdr.Id)); + bstrncpy(VolHdr.Id, "**error**", sizeof(VolHdr.Id)); /* Read ANSI/IBM label if so requested */ want_ansi_label = dcr->VolCatInfo.LabelType != B_BACULA_LABEL || dcr->device->label_type != B_BACULA_LABEL; - if (want_ansi_label || dev->has_cap(CAP_CHECKLABELS)) { + if (want_ansi_label || has_cap(CAP_CHECKLABELS)) { stat = read_ansi_ibm_label(dcr); /* If we want a label and didn't find it, return error */ if (want_ansi_label && stat != VOL_OK) { @@ -106,14 +106,14 @@ int read_dev_volume_label(DCR *dcr) } if (stat == VOL_NAME_ERROR || stat == VOL_LABEL_ERROR) { Mmsg(jcr->errmsg, _("Wrong Volume mounted on %s device %s: Wanted %s have %s\n"), - dev->print_type(), dev->print_name(), VolName, dev->VolHdr.VolumeName); - if (!dev->poll && jcr->label_errors++ > 100) { + print_type(), print_name(), VolName, VolHdr.VolumeName); + if (!poll && jcr->label_errors++ > 100) { Jmsg(jcr, M_FATAL, 0, _("Too many tries: %s"), jcr->errmsg); } goto bail_out; } if (stat != VOL_OK) { /* Not an ANSI/IBM label, so re-read */ - dev->rewind(dcr); + rewind(dcr); } else { have_ansi_label = true; } @@ -124,40 +124,48 @@ int read_dev_volume_label(DCR *dcr) empty_block(block); Dmsg0(130, "Big if statement in read_volume_label\n"); + dcr->reading_label = true; if (!dcr->read_block_from_dev(NO_BLOCK_NUMBER_CHECK)) { - Mmsg(jcr->errmsg, _("Requested Volume \"%s\" on %s device %s is not a Bacula " + Mmsg(jcr->errmsg, _("Read label block failed: requested Volume \"%s\" on %s device %s is not a Bacula " "labeled Volume, because: ERR=%s"), NPRT(VolName), - dev->print_type(), dev->print_name(), dev->print_errmsg()); - Dmsg1(130, "%s", jcr->errmsg); + print_type(), print_name(), print_errmsg()); + Dmsg1(dbglvl, "%s", jcr->errmsg); } else if (!read_record_from_block(dcr, record)) { Mmsg(jcr->errmsg, _("Could not read Volume label from block.\n")); - Dmsg1(130, "%s", jcr->errmsg); - } else if (!unser_volume_label(dev, record)) { + Dmsg1(dbglvl, "%s", jcr->errmsg); + } else if (!unser_volume_label(this, record)) { Mmsg(jcr->errmsg, _("Could not unserialize Volume label: ERR=%s\n"), - dev->print_errmsg()); - Dmsg1(130, "%s", jcr->errmsg); - } else if (strcmp(dev->VolHdr.Id, BaculaId) != 0 && - strcmp(dev->VolHdr.Id, OldBaculaId) != 0) { - Mmsg(jcr->errmsg, _("Volume Header Id bad: %s\n"), dev->VolHdr.Id); - Dmsg1(130, "%s", jcr->errmsg); + print_errmsg()); + Dmsg1(dbglvl, "%s", jcr->errmsg); + } else if (strcmp(VolHdr.Id, BaculaId) != 0 && + strcmp(VolHdr.Id, OldBaculaId) != 0 && + strcmp(VolHdr.Id, BaculaMetaDataId) != 0 && + strcmp(VolHdr.Id, BaculaAlignedDataId) != 0 && + strcmp(VolHdr.Id, BaculaS3CloudId) != 0) { + Mmsg(jcr->errmsg, _("Volume Header Id bad: %s\n"), VolHdr.Id); + Dmsg1(dbglvl, "%s", jcr->errmsg); } else { ok = true; + Dmsg1(dbglvl, "VolHdr.Id OK: %s\n", VolHdr.Id); } + dcr->reading_label = false; free_record(record); /* finished reading Volume record */ - if (!dev->is_volume_to_unload()) { - dev->clear_unload(); + if (!is_volume_to_unload()) { + clear_unload(); } if (!ok) { if (jcr->ignore_label_errors) { - dev->set_labeled(); /* set has Bacula label */ - Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg); + set_labeled(); /* set has Bacula label */ + if (jcr->errmsg[0]) { + Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg); + } empty_block(block); - Leave(100); + Leave(dbglvl); return VOL_OK; } - Dmsg0(100, "No volume label - bailing out\n"); + Dmsg0(dbglvl, "No volume label - bailing out\n"); stat = VOL_NO_LABEL; goto bail_out; } @@ -166,60 +174,91 @@ int read_dev_volume_label(DCR *dcr) * then read the Bacula Volume label. Now we need to * make sure we have the right Volume. */ - - - if (dev->VolHdr.VerNum != BaculaTapeVersion && - dev->VolHdr.VerNum != OldCompatibleBaculaTapeVersion1 && - dev->VolHdr.VerNum != OldCompatibleBaculaTapeVersion2) { + if (VolHdr.VerNum != BaculaTapeVersion && + VolHdr.VerNum != BaculaMetaDataVersion && + VolHdr.VerNum != BaculaS3CloudVersion && + VolHdr.VerNum != OldCompatibleBaculaTapeVersion1 && + VolHdr.VerNum != OldCompatibleBaculaTapeVersion2) { Mmsg(jcr->errmsg, _("Volume on %s device %s has wrong Bacula version. Wanted %d got %d\n"), - dev->print_type(), dev->print_name(), BaculaTapeVersion, dev->VolHdr.VerNum); - Dmsg1(130, "VOL_VERSION_ERROR: %s", jcr->errmsg); + print_type(), print_name(), BaculaTapeVersion, VolHdr.VerNum); + Dmsg1(dbglvl, "VOL_VERSION_ERROR: %s", jcr->errmsg); stat = VOL_VERSION_ERROR; goto bail_out; } + Dmsg1(dbglvl, "VolHdr.VerNum=%ld OK.\n", VolHdr.VerNum); /* We are looking for either an unused Bacula tape (PRE_LABEL) or * a Bacula volume label (VOL_LABEL) */ - if (dev->VolHdr.LabelType != PRE_LABEL && dev->VolHdr.LabelType != VOL_LABEL) { - Mmsg(jcr->errmsg, _("Volume on %s device %s has bad Bacula label type: %x\n"), - dev->print_type(), dev->print_name(), dev->VolHdr.LabelType); - Dmsg1(130, "%s", jcr->errmsg); - if (!dev->poll && jcr->label_errors++ > 100) { + if (VolHdr.LabelType != PRE_LABEL && VolHdr.LabelType != VOL_LABEL) { + Mmsg(jcr->errmsg, _("Volume on %s device %s has bad Bacula label type: %ld\n"), + print_type(), print_name(), VolHdr.LabelType); + Dmsg1(dbglvl, "%s", jcr->errmsg); + if (!poll && jcr->label_errors++ > 100) { Jmsg(jcr, M_FATAL, 0, _("Too many tries: %s"), jcr->errmsg); } - Dmsg0(150, "return VOL_LABEL_ERROR\n"); + Dmsg0(dbglvl, "return VOL_LABEL_ERROR\n"); stat = VOL_LABEL_ERROR; goto bail_out; } - dev->set_labeled(); /* set has Bacula label */ + set_labeled(); /* set has Bacula label */ /* Compare Volume Names */ - Dmsg2(130, "Compare Vol names: VolName=%s hdr=%s\n", VolName?VolName:"*", dev->VolHdr.VolumeName); - if (VolName && *VolName && *VolName != '*' && strcmp(dev->VolHdr.VolumeName, VolName) != 0) { + Dmsg2(130, "Compare Vol names: VolName=%s hdr=%s\n", VolName?VolName:"*", VolHdr.VolumeName); + if (VolName && *VolName && *VolName != '*' && strcmp(VolHdr.VolumeName, VolName) != 0) { Mmsg(jcr->errmsg, _("Wrong Volume mounted on %s device %s: Wanted %s have %s\n"), - dev->print_type(), dev->print_name(), VolName, dev->VolHdr.VolumeName); - Dmsg1(130, "%s", jcr->errmsg); + print_type(), print_name(), VolName, VolHdr.VolumeName); + Dmsg1(dbglvl, "%s", jcr->errmsg); /* * Cancel Job if too many label errors * => we are in a loop */ - if (!dev->poll && jcr->label_errors++ > 100) { + if (!poll && jcr->label_errors++ > 100) { Jmsg(jcr, M_FATAL, 0, "Too many tries: %s", jcr->errmsg); } - Dmsg0(150, "return VOL_NAME_ERROR\n"); + Dmsg0(dbglvl, "return VOL_NAME_ERROR\n"); stat = VOL_NAME_ERROR; goto bail_out; } + /* Compare VolType to Device Type */ + switch (dev_type) { + case B_FILE_DEV: + if (strcmp(VolHdr.Id, BaculaId) != 0) { + Mmsg(jcr->errmsg, _("Wrong Volume Type. Wanted a File or Tape Volume %s on device %s, but got: %s\n"), + VolHdr.VolumeName, print_name(), VolHdr.Id); + stat = VOL_TYPE_ERROR; + goto bail_out; + } + break; + case B_ALIGNED_DEV: + case B_ADATA_DEV: + if (strcmp(VolHdr.Id, BaculaMetaDataId) != 0) { + Mmsg(jcr->errmsg, _("Wrong Volume Type. Wanted an Aligned Volume %s on device %s, but got: %s\n"), + VolHdr.VolumeName, print_name(), VolHdr.Id); + stat = VOL_TYPE_ERROR; + goto bail_out; + } + break; + case B_CLOUD_DEV: + if (strcmp(VolHdr.Id, BaculaS3CloudId) != 0) { + Mmsg(jcr->errmsg, _("Wrong Volume Type. Wanted a Cloud Volume %s on device %s, but got: %s\n"), + VolHdr.VolumeName, print_name(), VolHdr.Id); + stat = VOL_TYPE_ERROR; + goto bail_out; + } + default: + break; + } + if (chk_dbglvl(100)) { - dump_volume_label(dev); + dump_volume_label(); } - Dmsg0(130, "Leave read_volume_label() VOL_OK\n"); + Dmsg0(dbglvl, "Leave read_volume_label() VOL_OK\n"); /* If we are a streaming device, we only get one chance to read */ - if (!dev->has_cap(CAP_STREAM)) { - dev->rewind(dcr); + if (!has_cap(CAP_STREAM)) { + rewind(dcr); if (have_ansi_label) { stat = read_ansi_ibm_label(dcr); /* If we want a label and didn't find it, return error */ @@ -229,27 +268,27 @@ int read_dev_volume_label(DCR *dcr) } } - Dmsg1(100, "Call reserve_volume=%s\n", dev->VolHdr.VolumeName); - if (reserve_volume(dcr, dev->VolHdr.VolumeName) == NULL) { + Dmsg1(100, "Call reserve_volume=%s\n", VolHdr.VolumeName); + if (reserve_volume(dcr, VolHdr.VolumeName) == NULL) { if (!jcr->errmsg[0]) { Mmsg3(jcr->errmsg, _("Could not reserve volume %s on %s device %s\n"), - dev->VolHdr.VolumeName, dev->print_type(), dev->print_name()); + VolHdr.VolumeName, print_type(), print_name()); } - Dmsg2(100, "Could not reserve volume %s on %s\n", dev->VolHdr.VolumeName, dev->print_name()); + Dmsg2(dbglvl, "Could not reserve volume %s on %s\n", VolHdr.VolumeName, print_name()); stat = VOL_NAME_ERROR; goto bail_out; } empty_block(block); - Leave(200); + Leave(dbglvl); return VOL_OK; bail_out: empty_block(block); - dev->rewind(dcr); - Dmsg1(150, "return %d\n", stat); - Leave(200); + rewind(dcr); + Dmsg2(dbglvl, "return stat=%d %s", stat, jcr->errmsg); + Leave(dbglvl); return stat; } @@ -260,26 +299,9 @@ bail_out: * Returns: false on failure * true on success * + * Handle both the ameta and adata volumes. */ -static bool write_volume_label_to_block(DCR *dcr) -{ - bool ok; - - Enter(100); - Dmsg0(130, "write Label in write_volume_label_to_block()\n"); - - Dmsg0(100, "Call sub_write_vol_label\n"); - ok = sub_write_volume_label_to_block(dcr); - if (!ok) { - goto get_out; - } - -get_out: - Leave(100); - return ok; -} - -static bool sub_write_volume_label_to_block(DCR *dcr) +bool DEVICE::write_volume_label_to_block(DCR *dcr) { DEVICE *dev; DEV_BLOCK *block; @@ -295,10 +317,11 @@ static bool sub_write_volume_label_to_block(DCR *dcr) memset(rec.data, 0, SER_LENGTH_Volume_Label); empty_block(block); /* Volume label always at beginning */ - create_volume_label_record(dcr, dcr->dev, &rec, false); + create_volume_label_record(dcr, dcr->dev, &rec, dcr->block->adata); block->BlockNumber = 0; - Dmsg0(100, "write_record_to_block\n"); + /* Note for adata this also writes to disk */ + Dmsg1(100, "write_record_to_block adata=%d\n", dcr->dev->adata); if (!write_record_to_block(dcr, &rec)) { free_pool_memory(rec.data); Jmsg2(jcr, M_FATAL, 0, _("Cannot write Volume label to block for %s device %s\n"), @@ -306,8 +329,8 @@ static bool sub_write_volume_label_to_block(DCR *dcr) ok = false; goto get_out; } else { - Dmsg3(100, "Wrote fd=%d label of %d bytes to block. Vol=%s\n", - dev->fd(), rec.data_len, dcr->VolumeName); + Dmsg4(100, "Wrote fd=%d adata=%d label of %d bytes to block. Vol=%s\n", + dev->fd(), block->adata, rec.data_len, dcr->VolumeName); } free_pool_memory(rec.data); @@ -326,13 +349,17 @@ get_out: * This routine should be used only when labeling a blank tape or * when recylcing a volume. * + * Handle both the ameta and adata volumes. */ -bool write_new_volume_label_to_dev(DCR *dcr, const char *VolName, - const char *PoolName, bool relabel, bool dvdnow) +bool DEVICE::write_volume_label(DCR *dcr, const char *VolName, + const char *PoolName, bool relabel, bool no_prelabel) { DEVICE *dev; Enter(100); + Dmsg4(230, "Write: block=%p ameta=%p dev=%p ameta_dev=%p\n", + dcr->block, dcr->ameta_block, dcr->dev, dcr->ameta_dev); + dcr->set_ameta(); dev = dcr->dev; Dmsg0(150, "write_volume_label()\n"); @@ -340,7 +367,7 @@ bool write_new_volume_label_to_dev(DCR *dcr, const char *VolName, if (dcr->jcr) { Mmsg(dcr->jcr->errmsg, "ERROR: new_volume_label_to_dev called with NULL VolName\n"); } - Pmsg0(0, "=== ERROR: write_new_volume_label_to_dev called with NULL VolName\n"); + Pmsg0(0, "=== ERROR: write_volume_label called with NULL VolName\n"); goto bail_out; } @@ -350,9 +377,7 @@ bool write_new_volume_label_to_dev(DCR *dcr, const char *VolName, if (!dev->truncate(dcr)) { goto bail_out; } - if (!dev->is_tape()) { - dev->close_part(dcr); /* make sure DVD/file closed for rename */ - } + dev->close_part(dcr); /* make sure closed for rename */ } /* Set the new filename for open, ... */ @@ -361,26 +386,27 @@ bool write_new_volume_label_to_dev(DCR *dcr, const char *VolName, dev->clearVolCatBytes(); Dmsg1(100, "New VolName=%s\n", VolName); - if (!dev->open(dcr, OPEN_READ_WRITE)) { + if (!dev->open_device(dcr, OPEN_READ_WRITE)) { /* If device is not tape, attempt to create it */ - if (dev->is_tape() || !dev->open(dcr, CREATE_READ_WRITE)) { - Jmsg4(dcr->jcr, M_WARNING, 0, _("Open %s device %s Volume \"%s\" failed: ERR=%s\n"), + if (dev->is_tape() || !dev->open_device(dcr, CREATE_READ_WRITE)) { + Jmsg4(dcr->jcr, M_WARNING, 0, _("Open %s device %s Volume \"%s\" failed: ERR=%s"), dev->print_type(), dev->print_name(), dcr->VolumeName, dev->bstrerror()); goto bail_out; } } Dmsg1(150, "Label type=%d\n", dev->label_type); - if (!sub_write_new_volume_label_to_dev(dcr, VolName, PoolName, relabel, dvdnow)) { + if (!write_volume_label_to_dev(dcr, VolName, PoolName, relabel, no_prelabel)) { goto bail_out; } - if (dev->weof(1)) { + + /* Not aligned data */ + if (dev->weof(dcr, 1)) { dev->set_labeled(); - write_ansi_ibm_labels(dcr, ANSI_EOF_LABEL, dev->VolHdr.VolumeName); } if (chk_dbglvl(100)) { - dump_volume_label(dev); + dev->dump_volume_label(); } Dmsg0(50, "Call reserve_volume\n"); /**** ***FIXME*** if dev changes, dcr must be updated */ @@ -398,20 +424,25 @@ bool write_new_volume_label_to_dev(DCR *dcr, const char *VolName, return true; bail_out: + dcr->adata_label = false; + dcr->set_ameta(); volume_unused(dcr); dcr->dev->clear_append(); /* remove append since this is PRE_LABEL */ Leave(100); return false; } -static bool sub_write_new_volume_label_to_dev(DCR *dcr, const char *VolName, - const char *PoolName, bool relabel, bool dvdnow) +bool DEVICE::write_volume_label_to_dev(DCR *dcr, const char *VolName, + const char *PoolName, bool relabel, bool no_prelabel) { - DEVICE *dev; + DEVICE *dev, *ameta_dev; DEV_BLOCK *block; + DEV_RECORD *rec = new_record(); + bool rtn = false; Enter(100); dev = dcr->dev; + ameta_dev = dcr->ameta_dev; block = dcr->block; empty_block(block); @@ -423,103 +454,115 @@ static bool sub_write_new_volume_label_to_dev(DCR *dcr, const char *VolName, /* Temporarily mark in append state to enable writing */ dev->set_append(); - /* Create PRE_LABEL or VOL_LABEL if DVD */ - create_volume_header(dev, VolName, PoolName, dvdnow); + /* Create PRE_LABEL or VOL_LABEL */ + create_volume_header(dev, VolName, PoolName, no_prelabel); /* * If we have already detected an ANSI label, re-read it * to skip past it. Otherwise, we write a new one if * so requested. */ - if (dev->label_type != B_BACULA_LABEL) { - if (read_ansi_ibm_label(dcr) != VOL_OK) { - dev->rewind(dcr); + if (!block->adata) { + if (dev->label_type != B_BACULA_LABEL) { + if (read_ansi_ibm_label(dcr) != VOL_OK) { + dev->rewind(dcr); + goto bail_out; + } + } else if (!write_ansi_ibm_labels(dcr, ANSI_VOL_LABEL, VolName)) { goto bail_out; } - } else if (!write_ansi_ibm_labels(dcr, ANSI_VOL_LABEL, VolName)) { - goto bail_out; } - create_volume_label_record(dcr, dev, dcr->rec, false); - dcr->rec->Stream = 0; - dcr->rec->maskedStream = 0; + create_volume_label_record(dcr, dev, rec, block->adata); + rec->Stream = 0; + rec->maskedStream = 0; - Dmsg1(100, "write_record_to_block FI=%d\n", dcr->rec->FileIndex); + Dmsg2(100, "write_record_to_block adata=%d FI=%d\n", dcr->dev->adata, + rec->FileIndex); - if (!write_record_to_block(dcr, dcr->rec)) { + /* For adata label this also writes to disk */ + if (!write_record_to_block(dcr, rec)) { Dmsg2(40, "Bad Label write on %s: ERR=%s\n", dev->print_name(), dev->print_errmsg()); goto bail_out; } else { - Dmsg2(100, "Wrote label=%d bytes block: %s\n", dcr->rec->data_len, dev->print_name()); + Dmsg3(100, "Wrote label=%d bytes adata=%d block: %s\n", rec->data_len, block->adata, dev->print_name()); } - Dmsg2(100, "New label VolCatBytes=%lld VolCatStatus=%s\n", - dev->VolCatInfo.VolCatBytes, dev->VolCatInfo.VolCatStatus); - - Dmsg3(130, "Call write_block_to_dev() fd=%d block=%p Addr=%lld\n", - dcr->dev->fd(), dcr->block, block->dev->lseek(dcr, 0, SEEK_CUR)); - Dmsg0(100, "write_record_to_dev\n"); - /* Write block to device */ - if (!dcr->write_block_to_dev()) { - Dmsg2(40, "Bad Label write on %s: ERR=%s\n", dev->print_name(), dev->print_errmsg()); - goto bail_out; + Dmsg3(100, "New label adata=%d VolCatBytes=%lld VolCatStatus=%s\n", + dev->adata, ameta_dev->VolCatInfo.VolCatBytes, ameta_dev->VolCatInfo.VolCatStatus); + + if (block->adata) { + /* Empty block and set data start address */ + empty_block(dcr->adata_block); + } else { + Dmsg4(130, "Call write_block_to_dev() fd=%d adata=%d block=%p Addr=%lld\n", + dcr->dev->fd(), dcr->block->adata, dcr->block, block->dev->lseek(dcr, 0, SEEK_CUR)); + Dmsg1(100, "write_record_to_dev adata=%d\n", dcr->dev->adata); + /* Write ameta block to device */ + if (!dcr->write_block_to_dev()) { + Dmsg2(40, "Bad Label write on %s: ERR=%s\n", dev->print_name(), dev->print_errmsg()); + goto bail_out; + } } - Dmsg2(100, "New label VolCatBytes=%lld VolCatStatus=%s\n", - dev->VolCatInfo.VolCatBytes, dev->VolCatInfo.VolCatStatus); - Leave(100); - return true; + Dmsg3(100, "Wrote new Vol label adata=%d VolCatBytes=%lld VolCatStatus=%s\n", + dev->adata, ameta_dev->VolCatInfo.VolCatBytes, ameta_dev->VolCatInfo.VolCatStatus); + rtn = true; bail_out: + free_record(rec); Leave(100); - return false; + return rtn; } /* * Write a volume label. This is ONLY called if we have a valid Bacula * label of type PRE_LABEL or we are recyling an existing Volume. * - * By calling write_volume_label_to_block + * By calling write_volume_label_to_block, both ameta and adata + * are updated. * * Returns: true if OK * false if unable to write it */ -bool DCR::rewrite_volume_label(bool recycle) +bool DEVICE::rewrite_volume_label(DCR *dcr, bool recycle) { - DCR *dcr = this; + char ed1[50]; + JCR *jcr = dcr->jcr; Enter(100); ASSERT2(dcr->VolumeName[0], "Empty Volume name"); - if (!dev->open(dcr, OPEN_READ_WRITE)) { + ASSERT(!dcr->block->adata); + if (!open_device(dcr, OPEN_READ_WRITE)) { Jmsg4(jcr, M_WARNING, 0, _("Open %s device %s Volume \"%s\" failed: ERR=%s\n"), - dev->print_type(), dev->print_name(), dcr->VolumeName, dev->bstrerror()); + print_type(), print_name(), dcr->VolumeName, bstrerror()); Leave(100); return false; } - Dmsg2(190, "set append found freshly labeled volume. fd=%d dev=%x\n", dev->fd(), dev); - dev->VolHdr.LabelType = VOL_LABEL; /* set Volume label */ - dev->set_append(); + Dmsg2(190, "set append found freshly labeled volume. fd=%d dev=%x\n", fd(), this); + VolHdr.LabelType = VOL_LABEL; /* set Volume label */ + set_append(); Dmsg0(100, "Rewrite_volume_label set volcatbytes=0\n"); - dev->clearVolCatBytes(); - dev->setVolCatStatus("Append"); /* set append status */ + clearVolCatBytes(); /* resets both ameta and adata byte counts */ + setVolCatStatus("Append"); /* set append status */ - if (!dev->has_cap(CAP_STREAM)) { - if (!dev->rewind(dcr)) { + if (!has_cap(CAP_STREAM)) { + if (!rewind(dcr)) { Jmsg3(jcr, M_FATAL, 0, _("Rewind error on %s device %s: ERR=%s\n"), - dev->print_type(), dev->print_name(), dev->print_errmsg()); + print_type(), print_name(), print_errmsg()); Leave(100); return false; } if (recycle) { Dmsg1(150, "Doing recycle. Vol=%s\n", dcr->VolumeName); - if (!dev->truncate(dcr)) { + if (!truncate(dcr)) { Jmsg3(jcr, M_FATAL, 0, _("Truncate error on %s device %s: ERR=%s\n"), - dev->print_type(), dev->print_name(), dev->print_errmsg()); + print_type(), print_name(), print_errmsg()); Leave(100); return false; } - if (!dev->open(dcr, OPEN_READ_WRITE)) { + if (!open_device(dcr, OPEN_READ_WRITE)) { Jmsg3(jcr, M_FATAL, 0, - _("Failed to re-open DVD after truncate on %s device %s: ERR=%s\n"), - dev->print_type(), dev->print_name(), dev->print_errmsg()); + _("Failed to re-open device after truncate on %s device %s: ERR=%s"), + print_type(), print_name(), print_errmsg()); Leave(100); return false; } @@ -531,10 +574,10 @@ bool DCR::rewrite_volume_label(bool recycle) Leave(100); return false; } - Dmsg1(100, "wrote vol label to block. Vol=%s\n", dcr->VolumeName); + Dmsg2(100, "wrote vol label to block. adata=%d Vol=%s\n", dcr->block->adata, dcr->VolumeName); ASSERT2(dcr->VolumeName[0], "Empty Volume name"); - dev->setVolCatInfo(false); + setVolCatInfo(false); /* * If we are not dealing with a streaming device, @@ -543,76 +586,80 @@ bool DCR::rewrite_volume_label(bool recycle) * We do not write the block now if this is an ANSI label. This * avoids re-writing the ANSI label, which we do not want to do. */ - if (!dev->has_cap(CAP_STREAM)) { + if (!has_cap(CAP_STREAM)) { /* * If we have already detected an ANSI label, re-read it * to skip past it. Otherwise, we write a new one if * so requested. */ - if (dev->label_type != B_BACULA_LABEL) { + if (label_type != B_BACULA_LABEL) { if (read_ansi_ibm_label(dcr) != VOL_OK) { - dev->rewind(dcr); + rewind(dcr); Leave(100); return false; } - } else if (!write_ansi_ibm_labels(dcr, ANSI_VOL_LABEL, dev->VolHdr.VolumeName)) { + } else if (!write_ansi_ibm_labels(dcr, ANSI_VOL_LABEL, VolHdr.VolumeName)) { Leave(100); return false; } /* Attempt write to check write permission */ - Dmsg1(200, "Attempt to write to device fd=%d.\n", dev->fd()); + Dmsg1(200, "Attempt to write to device fd=%d.\n", fd()); if (!dcr->write_block_to_dev()) { Jmsg3(jcr, M_ERROR, 0, _("Unable to write %s device %s: ERR=%s\n"), - dev->print_type(), dev->print_name(), dev->print_errmsg()); + print_type(), print_name(), print_errmsg()); Dmsg0(200, "===ERROR write block to dev\n"); Leave(100); return false; } } ASSERT2(dcr->VolumeName[0], "Empty Volume name"); - dev->setVolCatName(dcr->VolumeName); - if (!dir_get_volume_info(dcr, GET_VOL_INFO_FOR_WRITE)) { + setVolCatName(dcr->VolumeName); + if (!dir_get_volume_info(dcr, dcr->VolumeName, GET_VOL_INFO_FOR_WRITE)) { Leave(100); return false; } - dev->set_labeled(); + set_labeled(); /* Set or reset Volume statistics */ - dev->VolCatInfo.VolCatJobs = 0; - dev->VolCatInfo.VolCatFiles = 0; - dev->VolCatInfo.VolCatErrors = 0; - dev->VolCatInfo.VolCatBlocks = 0; - dev->VolCatInfo.VolCatRBytes = 0; + VolCatInfo.VolCatJobs = 0; + VolCatInfo.VolCatFiles = 0; + VolCatInfo.VolCatErrors = 0; + VolCatInfo.VolCatBlocks = 0; + VolCatInfo.VolCatRBytes = 0; + VolCatInfo.VolCatCloudParts = 0; + VolCatInfo.VolLastPartBytes = 0; + VolCatInfo.VolCatType = 0; /* Will be set by dir_update_volume_info() */ if (recycle) { - dev->VolCatInfo.VolCatMounts++; - dev->VolCatInfo.VolCatRecycles++; - dir_create_jobmedia_record(dcr, true); + VolCatInfo.VolCatMounts++; + VolCatInfo.VolCatRecycles++; } else { - dev->VolCatInfo.VolCatMounts = 1; - dev->VolCatInfo.VolCatRecycles = 0; - dev->VolCatInfo.VolCatWrites = 1; - dev->VolCatInfo.VolCatReads = 1; + VolCatInfo.VolCatMounts = 1; + VolCatInfo.VolCatRecycles = 0; + VolCatInfo.VolCatWrites = 1; + VolCatInfo.VolCatReads = 1; } + dcr->VolMediaId = dcr->VolCatInfo.VolMediaId; /* make create_jobmedia work */ + dir_create_jobmedia_record(dcr, true); Dmsg1(100, "dir_update_vol_info. Set Append vol=%s\n", dcr->VolumeName); - dev->VolCatInfo.VolFirstWritten = time(NULL); - dev->setVolCatStatus("Append"); + VolCatInfo.VolFirstWritten = time(NULL); + setVolCatStatus("Append"); if (!dir_update_volume_info(dcr, true, true)) { /* indicate relabel */ Leave(100); return false; } if (recycle) { Jmsg(jcr, M_INFO, 0, _("Recycled volume \"%s\" on %s device %s, all previous data lost.\n"), - dcr->VolumeName, dev->print_type(), dev->print_name()); + dcr->VolumeName, print_type(), print_name()); } else { Jmsg(jcr, M_INFO, 0, _("Wrote label to prelabeled Volume \"%s\" on %s device %s\n"), - dcr->VolumeName, dev->print_type(), dev->print_name()); + dcr->VolumeName, print_type(), print_name()); } /* * End writing real Volume label (from pre-labeled tape), or recycling * the volume. */ - Dmsg3(100, "OK from rewrite vol label. adata=%d slot=%d Vol=%s\n", - dcr->block->adata, dev->VolCatInfo.Slot, dcr->VolumeName); + Dmsg4(100, "OK rewrite vol label. Addr=%s adata=%d slot=%d Vol=%s\n", + print_addr(ed1, sizeof(ed1)), dcr->block->adata, VolCatInfo.Slot, dcr->VolumeName); Leave(100); return true; } @@ -620,12 +667,14 @@ bool DCR::rewrite_volume_label(bool recycle) /* * create_volume_label_record + * Note: it is assumed that you have created the volume_header + * (label) prior to calling this subroutine. * Serialize label (from dev->VolHdr structure) into device record. * Assumes that the dev->VolHdr structure is properly * initialized. */ static void create_volume_label_record(DCR *dcr, DEVICE *dev, - DEV_RECORD *rec, bool alt) + DEV_RECORD *rec, bool adata) { ser_declare; struct date_time dt; @@ -669,43 +718,79 @@ static void create_volume_label_record(DCR *dcr, DEVICE *dev, ser_string(dev->VolHdr.LabelProg); ser_string(dev->VolHdr.ProgVersion); ser_string(dev->VolHdr.ProgDate); + /* ***FIXME*** */ + dev->VolHdr.AlignedVolumeName[0] = 0; + ser_string(dev->VolHdr.AlignedVolumeName); + + /* This is adata Volume information */ + ser_uint64(dev->VolHdr.FirstData); + ser_uint32(dev->VolHdr.FileAlignment); + ser_uint32(dev->VolHdr.PaddingSize); + /* adata and dedup volumes */ + ser_uint32(dev->VolHdr.BlockSize); ser_end(rec->data, SER_LENGTH_Volume_Label); - bstrncpy(dcr->VolumeName, dev->VolHdr.VolumeName, sizeof(dcr->VolumeName)); + if (!adata) { + bstrncpy(dcr->VolumeName, dev->VolHdr.VolumeName, sizeof(dcr->VolumeName)); + } ASSERT2(dcr->VolumeName[0], "Empty Volume name"); rec->data_len = ser_length(rec->data); rec->FileIndex = dev->VolHdr.LabelType; - Dmsg1(100, "LabelType=%d\n", dev->VolHdr.LabelType); + Dmsg2(100, "LabelType=%d adata=%d\n", dev->VolHdr.LabelType, dev->adata); rec->VolSessionId = jcr->VolSessionId; rec->VolSessionTime = jcr->VolSessionTime; rec->Stream = jcr->NumWriteVolumes; rec->maskedStream = jcr->NumWriteVolumes; - Dmsg2(100, "Created Vol label rec: FI=%s len=%d\n", FI_to_ascii(buf, rec->FileIndex), + Dmsg3(100, "Created adata=%d Vol label rec: FI=%s len=%d\n", adata, FI_to_ascii(buf, rec->FileIndex), rec->data_len); - Dmsg2(100, "reclen=%d recdata=%s\n", rec->data_len, rec->data); + Dmsg2(100, "reclen=%d recdata=%s", rec->data_len, rec->data); Leave(100); } /* - * Create a volume header in memory + * Create a volume header (label) in memory + * The volume record is created after this header (label) + * is created. */ void create_volume_header(DEVICE *dev, const char *VolName, - const char *PoolName, bool dvdnow) + const char *PoolName, bool no_prelabel) { DEVRES *device = (DEVRES *)dev->device; - Dmsg0(130, "Start create_volume_header()\n"); - - ASSERT(dev != NULL); + Enter(130); + + ASSERT2(dev != NULL, "dev ptr is NULL"); + + if (dev->is_aligned()) { + bstrncpy(dev->VolHdr.Id, BaculaMetaDataId, sizeof(dev->VolHdr.Id)); + dev->VolHdr.VerNum = BaculaMetaDataVersion; + dev->VolHdr.FirstData = dev->file_alignment; + dev->VolHdr.FileAlignment = dev->file_alignment; + dev->VolHdr.PaddingSize = dev->padding_size; + dev->VolHdr.BlockSize = dev->adata_size; + } else if (dev->is_adata()) { + bstrncpy(dev->VolHdr.Id, BaculaAlignedDataId, sizeof(dev->VolHdr.Id)); + dev->VolHdr.VerNum = BaculaAlignedDataVersion; + dev->VolHdr.FirstData = dev->file_alignment; + dev->VolHdr.FileAlignment = dev->file_alignment; + dev->VolHdr.PaddingSize = dev->padding_size; + dev->VolHdr.BlockSize = dev->adata_size; + } else if (dev->is_cloud()) { + bstrncpy(dev->VolHdr.Id, BaculaS3CloudId, sizeof(dev->VolHdr.Id)); + dev->VolHdr.VerNum = BaculaS3CloudVersion; + dev->VolHdr.BlockSize = dev->max_block_size; + } else { + bstrncpy(dev->VolHdr.Id, BaculaId, sizeof(dev->VolHdr.Id)); + dev->VolHdr.VerNum = BaculaTapeVersion; + dev->VolHdr.BlockSize = dev->max_block_size; + } - bstrncpy(dev->VolHdr.Id, BaculaId, sizeof(dev->VolHdr.Id)); - dev->VolHdr.VerNum = BaculaTapeVersion; - if (dev->is_dvd() && dvdnow) { - /* We do not want to re-label a DVD so write VOL_LABEL now */ + if (dev->has_cap(CAP_STREAM) && no_prelabel) { + /* We do not want to re-label so write VOL_LABEL now */ dev->VolHdr.LabelType = VOL_LABEL; } else { - dev->VolHdr.LabelType = PRE_LABEL; /* Mark tape as unused */ + dev->VolHdr.LabelType = PRE_LABEL; /* Mark Volume as unused */ } bstrncpy(dev->VolHdr.VolumeName, VolName, sizeof(dev->VolHdr.VolumeName)); bstrncpy(dev->VolHdr.PoolName, PoolName, sizeof(dev->VolHdr.PoolName)); @@ -725,12 +810,12 @@ void create_volume_header(DEVICE *dev, const char *VolName, sprintf(dev->VolHdr.ProgDate, "Build %s %s ", __DATE__, __TIME__); dev->set_labeled(); /* set has Bacula label */ if (chk_dbglvl(100)) { - dump_volume_label(dev); + dev->dump_volume_label(); } } /* - * Create session label + * Create session (Job) label * The pool memory must be released by the calling program */ void create_session_label(DCR *dcr, DEV_RECORD *rec, int label) @@ -771,10 +856,10 @@ void create_session_label(DCR *dcr, DEV_RECORD *rec, int label) if (label == EOS_LABEL) { ser_uint32(jcr->JobFiles); ser_uint64(jcr->JobBytes); - ser_uint32(dcr->StartBlock); - ser_uint32(dcr->EndBlock); - ser_uint32(dcr->StartFile); - ser_uint32(dcr->EndFile); + ser_uint32((uint32_t)dcr->StartAddr); /* Start Block */ + ser_uint32((uint32_t)dcr->EndAddr); /* End Block */ + ser_uint32((uint32_t)(dcr->StartAddr>>32)); /* Start File */ + ser_uint32((uint32_t)(dcr->EndAddr>>32)); /* End File */ ser_uint32(jcr->JobErrors); /* Added in VerNum 11 */ @@ -785,7 +870,7 @@ void create_session_label(DCR *dcr, DEV_RECORD *rec, int label) Leave(100); } -/* Write session label +/* Write session (Job) label * Returns: false on failure * true on success */ @@ -798,6 +883,14 @@ bool write_session_label(DCR *dcr, int label) char buf1[100], buf2[100]; Enter(100); + dev->Lock(); + Dmsg2(140, "=== write_session_label label=%d Vol=%s.\n", label, dev->getVolCatName()); + if (!check_for_newvol_or_newfile(dcr)) { + Pmsg0(000, "ERR: !check_for_new_vol_or_newfile\n"); + dev->Unlock(); + return false; + } + rec = new_record(); Dmsg1(130, "session_label record=%x\n", rec); switch (label) { @@ -805,20 +898,16 @@ bool write_session_label(DCR *dcr, int label) set_start_vol_position(dcr); break; case EOS_LABEL: - if (dev->is_tape()) { - dcr->EndBlock = dev->EndBlock; - dcr->EndFile = dev->EndFile; - } else { - dcr->EndBlock = (uint32_t)dev->file_addr; - dcr->EndFile = (uint32_t)(dev->file_addr >> 32); - } + dcr->EndAddr = dev->get_full_addr(); break; default: Jmsg1(jcr, M_ABORT, 0, _("Bad Volume session label request=%d\n"), label); break; } + create_session_label(dcr, rec, label); rec->FileIndex = label; + dev->Unlock(); /* * We guarantee that the session record can totally fit @@ -893,9 +982,9 @@ bool unser_volume_label(DEVICE *dev, DEV_RECORD *rec) /* Unserialize the record into the Volume Header */ - Dmsg2(100, "reclen=%d recdata=%s\n", rec->data_len, rec->data); + Dmsg2(100, "reclen=%d recdata=%s", rec->data_len, rec->data); rec->data = check_pool_memory_size(rec->data, SER_LENGTH_Volume_Label); - Dmsg2(100, "reclen=%d recdata=%s\n", rec->data_len, rec->data); + Dmsg2(100, "reclen=%d recdata=%s", rec->data_len, rec->data); ser_begin(rec->data, SER_LENGTH_Volume_Label); unser_string(dev->VolHdr.Id); unser_uint32(dev->VolHdr.VerNum); @@ -921,10 +1010,17 @@ bool unser_volume_label(DEVICE *dev, DEV_RECORD *rec) unser_string(dev->VolHdr.ProgVersion); unser_string(dev->VolHdr.ProgDate); +// unser_string(dev->VolHdr.AlignedVolumeName); + dev->VolHdr.AlignedVolumeName[0] = 0; + unser_uint64(dev->VolHdr.FirstData); + unser_uint32(dev->VolHdr.FileAlignment); + unser_uint32(dev->VolHdr.PaddingSize); + unser_uint32(dev->VolHdr.BlockSize); + ser_end(rec->data, SER_LENGTH_Volume_Label); Dmsg0(190, "unser_vol_label\n"); if (chk_dbglvl(100)) { - dump_volume_label(dev); + dev->dump_volume_label(); } Leave(100); return true; @@ -980,7 +1076,7 @@ bool unser_session_label(SESSION_LABEL *label, DEV_RECORD *rec) return true; } -void dump_volume_label(DEVICE *dev) +void DEVICE::dump_volume_label() { int64_t dbl = debug_level; uint32_t File; @@ -990,8 +1086,8 @@ void dump_volume_label(DEVICE *dev) struct date_time dt; debug_level = 1; - File = dev->file; - switch (dev->VolHdr.LabelType) { + File = file; + switch (VolHdr.LabelType) { case PRE_LABEL: LabelType = "PRE_LABEL"; break; @@ -1011,11 +1107,12 @@ void dump_volume_label(DEVICE *dev) goto bail_out; default: LabelType = buf; - sprintf(buf, _("Unknown %d"), dev->VolHdr.LabelType); + sprintf(buf, _("Unknown %d"), VolHdr.LabelType); break; } - Pmsg11(-1, _("\nVolume Label:\n" + Pmsg12(-1, _("\nVolume Label:\n" +"Adata : %d\n" "Id : %s" "VerNo : %d\n" "VolName : %s\n" @@ -1028,19 +1125,20 @@ void dump_volume_label(DEVICE *dev) "PoolType : %s\n" "HostName : %s\n" ""), - dev->VolHdr.Id, dev->VolHdr.VerNum, - dev->VolHdr.VolumeName, dev->VolHdr.PrevVolumeName, - File, LabelType, dev->VolHdr.LabelSize, - dev->VolHdr.PoolName, dev->VolHdr.MediaType, - dev->VolHdr.PoolType, dev->VolHdr.HostName); - - if (dev->VolHdr.VerNum >= 11) { + adata, + VolHdr.Id, VolHdr.VerNum, + VolHdr.VolumeName, VolHdr.PrevVolumeName, + File, LabelType, VolHdr.LabelSize, + VolHdr.PoolName, VolHdr.MediaType, + VolHdr.PoolType, VolHdr.HostName); + + if (VolHdr.VerNum >= 11) { char dt[50]; - bstrftime(dt, sizeof(dt), btime_to_utime(dev->VolHdr.label_btime)); + bstrftime(dt, sizeof(dt), btime_to_utime(VolHdr.label_btime)); Pmsg1(-1, _("Date label written: %s\n"), dt); } else { - dt.julian_day_number = dev->VolHdr.label_date; - dt.julian_day_fraction = dev->VolHdr.label_time; + dt.julian_day_number = VolHdr.label_date; + dt.julian_day_fraction = VolHdr.label_time; tm_decode(&dt, &tm); Pmsg5(-1, _("Date label written: %04d-%02d-%02d at %02d:%02d\n"), @@ -1217,7 +1315,7 @@ int dump_label_record(DEVICE *dev, DEV_RECORD *rec, int verbose, bool check_err) case PRE_LABEL: case VOL_LABEL: unser_volume_label(dev, rec); - dump_volume_label(dev); + dev->dump_volume_label(); break; case EOS_LABEL: diff --git a/bacula/src/stored/lock.c b/bacula/src/stored/lock.c index f269bcec7b..aebc31a787 100644 --- a/bacula/src/stored/lock.c +++ b/bacula/src/stored/lock.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -154,25 +154,26 @@ void DEVICE::dunblock(bool locked) #ifdef DEV_DEBUG_LOCK -void DEVICE::dbg_Lock(const char *tfile, int line) +void DEVICE::dbg_Lock(const char *file, int line) { - Dmsg3(sd_dbglvl, "Lock from %s:%d precnt=%d\n", tfile, line, m_count); - bthread_mutex_lock_p(&m_mutex, tfile, line); + Dmsg4(sd_dbglvl, "Lock %s from %s:%d precnt=%d\n", device->hdr.name, file, line, m_count); + bthread_mutex_lock_p(&m_mutex, file, line); m_pid = pthread_self(); m_count++; } -void DEVICE::dbg_Unlock(const char *tfile, int line) +void DEVICE::dbg_Unlock(const char *file, int line) { m_count--; - Dmsg3(sd_dbglvl, "Unlock from %s:%d postcnt=%d\n", tfile, line, m_count); - bthread_mutex_unlock_p(&m_mutex, tfile, line); + clear_thread_id(m_pid); + Dmsg4(sd_dbglvl, "Unlock %s from %s:%d postcnt=%d\n", device->hdr.name, file, line, m_count); + bthread_mutex_unlock_p(&m_mutex, file, line); } -void DEVICE::dbg_rUnlock(const char *tfile, int line) +void DEVICE::dbg_rUnlock(const char *file, int line) { - Dmsg2(sd_dbglvl, "rUnlock from %s:%d\n", tfile, line); - dbg_Unlock(tfile, line); + Dmsg2(sd_dbglvl, "rUnlock from %s:%d\n", file, line); + dbg_Unlock(file, line); } #else @@ -208,14 +209,15 @@ void DEVICE::Unlock() * and preparing the label. */ #ifdef DEV_DEBUG_LOCK -void DEVICE::dbg_rLock(const char *tfile, int line, bool locked) +void DEVICE::dbg_rLock(const char *file, int line, bool locked) { - Dmsg3(sd_dbglvl, "rLock blked=%s from %s:%d\n", print_blocked(), - tfile, line); - + Dmsg3(sd_dbglvl, "Enter rLock blked=%s from %s:%d\n", print_blocked(), + file, line); if (!locked) { /* lockmgr version of P(m_mutex) */ - bthread_mutex_lock_p(&m_mutex, tfile, line); + Dmsg4(sd_dbglvl, "Lock %s in rLock %s from %s:%d\n", + device->hdr.name, print_blocked(), file, line); + bthread_mutex_lock_p(&m_mutex, file, line); m_count++; } @@ -225,12 +227,12 @@ void DEVICE::dbg_rLock(const char *tfile, int line, bool locked) int stat; #ifndef HAVE_WIN32 /* thread id on Win32 may be a struct */ - Dmsg3(sd_dbglvl, "rLock blked=%s no_wait=%p me=%p\n", print_blocked(), - no_wait_id, pthread_self()); + Dmsg5(sd_dbglvl, "Blocked by %d %s in rLock blked=%s no_wait=%p me=%p\n", + blocked_by, device->hdr.name, print_blocked(), no_wait_id, pthread_self()); #endif - if ((stat = bthread_cond_wait_p(&this->wait, &m_mutex, tfile, line)) != 0) { + if ((stat = bthread_cond_wait_p(&this->wait, &m_mutex, file, line)) != 0) { berrno be; - this->dbg_Unlock(tfile, line); + this->dbg_Unlock(file, line); Emsg1(M_ABORT, 0, _("pthread_cond_wait failure. ERR=%s\n"), be.bstrerror(stat)); } @@ -242,7 +244,6 @@ void DEVICE::dbg_rLock(const char *tfile, int line, bool locked) void DEVICE::rLock(bool locked) { - if (!locked) { Lock(); m_count++; @@ -254,8 +255,8 @@ void DEVICE::rLock(bool locked) int stat; #ifndef HAVE_WIN32 /* thread id on Win32 may be a struct */ - Dmsg3(sd_dbglvl, "rLock blked=%s no_wait=%p me=%p\n", print_blocked(), - no_wait_id, pthread_self()); + Dmsg5(sd_dbglvl, "Blocked by %d rLock %s blked=%s no_wait=%p me=%p\n", + blocked_by, device->hdr.name, print_blocked(), no_wait_id, pthread_self()); #endif if ((stat = pthread_cond_wait(&this->wait, &m_mutex)) != 0) { berrno be; @@ -272,38 +273,38 @@ void DEVICE::rLock(bool locked) #ifdef SD_DEBUG_LOCK -void DEVICE::dbg_Lock_acquire(const char *tfile, int line) +void DEVICE::dbg_Lock_acquire(const char *file, int line) { - Dmsg2(sd_dbglvl, "Lock_acquire from %s:%d\n", tfile, line); - bthread_mutex_lock_p(&acquire_mutex, tfile, line); + Dmsg2(sd_dbglvl, "Lock_acquire from %s:%d\n", file, line); + bthread_mutex_lock_p(&acquire_mutex, file, line); } -void DEVICE::dbg_Unlock_acquire(const char *tfile, int line) +void DEVICE::dbg_Unlock_acquire(const char *file, int line) { - Dmsg2(sd_dbglvl, "Unlock_acquire from %s:%d\n", tfile, line); - bthread_mutex_unlock_p(&acquire_mutex, tfile, line); + Dmsg2(sd_dbglvl, "Unlock_acquire from %s:%d\n", file, line); + bthread_mutex_unlock_p(&acquire_mutex, file, line); } -void DEVICE::dbg_Lock_read_acquire(const char *tfile, int line) +void DEVICE::dbg_Lock_read_acquire(const char *file, int line) { - Dmsg2(sd_dbglvl, "Lock_read_acquire from %s:%d\n", tfile, line); - bthread_mutex_lock_p(&read_acquire_mutex, tfile, line); + Dmsg2(sd_dbglvl, "Lock_read_acquire from %s:%d\n", file, line); + bthread_mutex_lock_p(&read_acquire_mutex, file, line); } -void DEVICE::dbg_Unlock_read_acquire(const char *tfile, int line) +void DEVICE::dbg_Unlock_read_acquire(const char *file, int line) { - Dmsg2(sd_dbglvl, "Unlock_read_acquire from %s:%d\n", tfile, line); - bthread_mutex_unlock_p(&read_acquire_mutex, tfile, line); + Dmsg2(sd_dbglvl, "Unlock_read_acquire from %s:%d\n", file, line); + bthread_mutex_unlock_p(&read_acquire_mutex, file, line); } -void DEVICE::dbg_Lock_VolCatInfo(const char *tfile, int line) +void DEVICE::dbg_Lock_VolCatInfo(const char *file, int line) { - bthread_mutex_lock_p(&volcat_mutex, tfile, line); + bthread_mutex_lock_p(&volcat_mutex, file, line); } -void DEVICE::dbg_Unlock_VolCatInfo(const char *tfile, int line) +void DEVICE::dbg_Unlock_VolCatInfo(const char *file, int line) { - bthread_mutex_unlock_p(&volcat_mutex, tfile, line); + bthread_mutex_unlock_p(&volcat_mutex, file, line); } #else @@ -406,7 +407,9 @@ void _block_device(const char *file, int line, DEVICE *dev, int state) ASSERT2(dev->blocked() == BST_NOT_BLOCKED, "Block request of device already blocked"); dev->set_blocked(state); /* make other threads wait */ dev->no_wait_id = pthread_self(); /* allow us to continue */ - Dmsg3(sd_dbglvl, "set blocked=%s from %s:%d\n", dev->print_blocked(), file, line); + dev->blocked_by = get_jobid_from_tsd(); + Dmsg4(sd_dbglvl, "Blocked %s %s from %s:%d\n", + dev->device->hdr.name, dev->print_blocked(), file, line); } /* @@ -416,9 +419,11 @@ void _block_device(const char *file, int line, DEVICE *dev, int state) */ void _unblock_device(const char *file, int line, DEVICE *dev) { - Dmsg3(sd_dbglvl, "unblock %s from %s:%d\n", dev->print_blocked(), file, line); + Dmsg4(sd_dbglvl, "Unblocked %s %s from %s:%d\n", dev->device->hdr.name, + dev->print_blocked(), file, line); ASSERT2(dev->blocked(), "Unblock request of device not blocked"); dev->set_blocked(BST_NOT_BLOCKED); + dev->blocked_by = 0; clear_thread_id(dev->no_wait_id); if (dev->num_waiting > 0) { pthread_cond_broadcast(&dev->wait); /* wake them up */ @@ -431,14 +436,16 @@ void _unblock_device(const char *file, int line, DEVICE *dev) */ void _steal_device_lock(const char *file, int line, DEVICE *dev, bsteal_lock_t *hold, int state) { - Dmsg3(sd_dbglvl, "steal lock. old=%s from %s:%d\n", dev->print_blocked(), - file, line); + Dmsg4(sd_dbglvl, "Steal lock %s old=%s from %s:%d\n", + dev->device->hdr.name, dev->print_blocked(), file, line); hold->dev_blocked = dev->blocked(); hold->dev_prev_blocked = dev->dev_prev_blocked; hold->no_wait_id = dev->no_wait_id; + hold->blocked_by = dev->blocked_by; dev->set_blocked(state); Dmsg1(sd_dbglvl, "steal lock. new=%s\n", dev->print_blocked()); dev->no_wait_id = pthread_self(); + dev->blocked_by = get_jobid_from_tsd(); dev->Unlock(); } @@ -448,12 +455,13 @@ void _steal_device_lock(const char *file, int line, DEVICE *dev, bsteal_lock_t * */ void _give_back_device_lock(const char *file, int line, DEVICE *dev, bsteal_lock_t *hold) { - Dmsg3(sd_dbglvl, "return lock. old=%s from %s:%d\n", - dev->print_blocked(), file, line); + Dmsg4(sd_dbglvl, "Return lock %s old=%s from %s:%d\n", + dev->device->hdr.name, dev->print_blocked(), file, line); dev->Lock(); dev->set_blocked(hold->dev_blocked); dev->dev_prev_blocked = hold->dev_prev_blocked; dev->no_wait_id = hold->no_wait_id; + dev->blocked_by = hold->blocked_by; Dmsg1(sd_dbglvl, "return lock. new=%s\n", dev->print_blocked()); if (dev->num_waiting > 0) { pthread_cond_broadcast(&dev->wait); /* wake them up */ diff --git a/bacula/src/stored/lock.h b/bacula/src/stored/lock.h index e795d0ebfa..bc0a3a286b 100644 --- a/bacula/src/stored/lock.h +++ b/bacula/src/stored/lock.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -108,6 +108,7 @@ typedef struct s_steal_lock { pthread_t no_wait_id; /* id of no wait thread */ int dev_blocked; /* state */ int dev_prev_blocked; /* previous blocked state */ + uint32_t blocked_by; /* previous blocker */ } bsteal_lock_t; /* diff --git a/bacula/src/stored/match_bsr.c b/bacula/src/stored/match_bsr.c index 85cd565b0b..6f5548ffb9 100644 --- a/bacula/src/stored/match_bsr.c +++ b/bacula/src/stored/match_bsr.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -41,7 +41,10 @@ #include "lib/fnmatch.h" #endif -const int dbglevel = 500; +/* Temp code to test the new match_all code */ +int use_new_match_all = 0; + +const int dbglevel = 200; /* Forward references */ static int match_volume(BSR *bsr, BSR_VOLUME *volume, VOLUME_LABEL *volrec, bool done); @@ -52,8 +55,7 @@ static int match_job(BSR *bsr, BSR_JOB *job, SESSION_LABEL *sessrec, bool done); static int match_job_type(BSR *bsr, BSR_JOBTYPE *job_type, SESSION_LABEL *sessrec, bool done); static int match_job_level(BSR *bsr, BSR_JOBLEVEL *job_level, SESSION_LABEL *sessrec, bool done); static int match_jobid(BSR *bsr, BSR_JOBID *jobid, SESSION_LABEL *sessrec, bool done); -static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, bool done); -static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, bool done); +static int match_findex(BSR *bsr, DEV_RECORD *rec, bool done); static int match_voladdr(BSR *bsr, BSR_VOLADDR *voladdr, DEV_RECORD *rec, bool done); static int match_stream(BSR *bsr, BSR_STREAM *stream, DEV_RECORD *rec, bool done); static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, bool done, JCR *jcr); @@ -61,6 +63,9 @@ static int match_block_sesstime(BSR *bsr, BSR_SESSTIME *sesstime, DEV_BLOCK *blo static int match_block_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_BLOCK *block); static BSR *find_smallest_volfile(BSR *fbsr, BSR *bsr); +/* Temp function to test the new code */ +static int new_match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, + SESSION_LABEL *sessrec, bool done, JCR *jcr); /********************************************************************* * @@ -177,9 +182,19 @@ int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *se * In this case, we can probably reposition the * tape to the next available bsr position. */ + if (jcr->use_new_match_all) { /* TODO: Remove the if when the new code is tested */ + if (bsr->cur_bsr) { + bsr = bsr->cur_bsr; + } + } if (bsr) { bsr->reposition = false; - stat = match_all(bsr, rec, volrec, sessrec, true, jcr); + /* Temp code to test the new match_all */ + if (jcr->use_new_match_all) { + stat = new_match_all(bsr, rec, volrec, sessrec, true, jcr); + } else { + stat = match_all(bsr, rec, volrec, sessrec, true, jcr); + } /* * Note, bsr->reposition is set by match_all when * a bsr is done. We turn it off if a match was @@ -274,10 +289,6 @@ static bool get_smallest_voladdr(BSR_VOLADDR *va, uint64_t *ret) static BSR *find_smallest_volfile(BSR *found_bsr, BSR *bsr) { BSR *return_bsr = found_bsr; - BSR_VOLFILE *vf; - BSR_VOLBLOCK *vb; - uint32_t found_bsr_sfile, bsr_sfile; - uint32_t found_bsr_sblock, bsr_sblock; uint64_t found_bsr_saddr, bsr_saddr; /* if we have VolAddr, use it, else try with File and Block */ @@ -291,50 +302,6 @@ static BSR *find_smallest_volfile(BSR *found_bsr, BSR *bsr) } } - /* Find the smallest file in the found_bsr */ - vf = found_bsr->volfile; - found_bsr_sfile = vf->sfile; - while ( (vf=vf->next) ) { - if (vf->sfile < found_bsr_sfile) { - found_bsr_sfile = vf->sfile; - } - } - - /* Find the smallest file in the bsr */ - vf = bsr->volfile; - bsr_sfile = vf->sfile; - while ( (vf=vf->next) ) { - if (vf->sfile < bsr_sfile) { - bsr_sfile = vf->sfile; - } - } - - /* if the bsr file is less than the found_bsr file, return bsr */ - if (found_bsr_sfile > bsr_sfile) { - return_bsr = bsr; - } else if (found_bsr_sfile == bsr_sfile) { - /* Files are equal */ - /* find smallest block in found_bsr */ - vb = found_bsr->volblock; - found_bsr_sblock = vb->sblock; - while ( (vb=vb->next) ) { - if (vb->sblock < found_bsr_sblock) { - found_bsr_sblock = vb->sblock; - } - } - /* Find smallest block in bsr */ - vb = bsr->volblock; - bsr_sblock = vb->sblock; - while ( (vb=vb->next) ) { - if (vb->sblock < bsr_sblock) { - bsr_sblock = vb->sblock; - } - } - /* Compare and return the smallest */ - if (found_bsr_sblock > bsr_sblock) { - return_bsr = bsr; - } - } return return_bsr; } @@ -348,7 +315,7 @@ static BSR *find_smallest_volfile(BSR *found_bsr, BSR *bsr) * Returns: true if we should reposition * : false otherwise. */ -bool is_this_bsr_done(BSR *bsr, DEV_RECORD *rec) +bool is_this_bsr_done(JCR *jcr, BSR *bsr, DEV_RECORD *rec) { BSR *rbsr = rec->bsr; Dmsg1(dbglevel, "match_set %d\n", rbsr != NULL); @@ -356,19 +323,169 @@ bool is_this_bsr_done(BSR *bsr, DEV_RECORD *rec) return false; } rec->bsr = NULL; - rbsr->found++; - if (rbsr->count && rbsr->found >= rbsr->count) { - rbsr->done = true; - rbsr->root->reposition = true; - Dmsg2(dbglevel, "is_end_this_bsr set reposition=1 count=%d found=%d\n", - rbsr->count, rbsr->found); - return true; + + /* TODO: When the new code is stable, drop the else part */ + if (jcr->use_new_match_all) { + if (!rbsr->next) { + rbsr->found++; + } + /* Normally the loop must stop only *after* the last record has been read, + * and we are about to read the next record. + */ + if (rbsr->count && rbsr->found > rbsr->count) { + rbsr->done = true; + rbsr->root->reposition = true; + Dmsg2(dbglevel, "is_end_this_bsr set reposition=1 count=%d found=%d\n", + rbsr->count, rbsr->found); + return true; + } + + } else { + /* Old code that is stable */ + rbsr->found++; + + if (rbsr->count && rbsr->found >= rbsr->count) { + rbsr->done = true; + rbsr->root->reposition = true; + Dmsg2(dbglevel, "is_end_this_bsr set reposition=1 count=%d found=%d\n", + rbsr->count, rbsr->found); + return true; + } } Dmsg2(dbglevel, "is_end_this_bsr not done count=%d found=%d\n", rbsr->count, rbsr->found); return false; } + +static int new_match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, + SESSION_LABEL *sessrec, bool done, JCR *jcr) +{ + Dmsg0(dbglevel, "Enter match_all\n"); + for ( ; bsr; ) { + if (bsr->done) { + goto no_match; + } + if (!match_volume(bsr, bsr->volume, volrec, 1)) { + Dmsg2(dbglevel, "bsr fail bsr_vol=%s != rec read_vol=%s\n", bsr->volume->VolumeName, + volrec->VolumeName); + goto no_match; + } + + if (!match_voladdr(bsr, bsr->voladdr, rec, 1)) { + if (bsr->voladdr) { + Dmsg3(dbglevel, "Fail on Addr=%llu. bsr=%llu,%llu\n", + get_record_address(rec), bsr->voladdr->saddr, bsr->voladdr->eaddr); + } + goto no_match; + } + + if (!match_sesstime(bsr, bsr->sesstime, rec, 1)) { + Dmsg2(dbglevel, "Fail on sesstime. bsr=%u rec=%u\n", + bsr->sesstime->sesstime, rec->VolSessionTime); + goto no_match; + } + + /* NOTE!! This test MUST come after the sesstime test */ + if (!match_sessid(bsr, bsr->sessid, rec)) { + Dmsg2(dbglevel, "Fail on sessid. bsr=%u rec=%u\n", + bsr->sessid->sessid, rec->VolSessionId); + goto no_match; + } + + /* NOTE!! This test MUST come after sesstime and sessid tests */ + if (!match_findex(bsr, rec, 1)) { + Dmsg3(dbglevel, "Fail on recFI=%d. bsrFI=%d,%d\n", + rec->FileIndex, bsr->FileIndex->findex, bsr->FileIndex->findex2); + goto no_match; + } + if (bsr->FileIndex) { + Dmsg3(dbglevel, "match on findex=%d. bsrFI=%d,%d\n", + rec->FileIndex, bsr->FileIndex->findex, bsr->FileIndex->findex2); + } + + if (!match_fileregex(bsr, rec, jcr)) { + Dmsg1(dbglevel, "Fail on fileregex='%s'\n", NPRT(bsr->fileregex)); + goto no_match; + } + + /* This flag is set by match_fileregex (and perhaps other tests) */ + if (bsr->skip_file) { + Dmsg1(dbglevel, "Skipping findex=%d\n", rec->FileIndex); + goto no_match; + } + + /* + * If a count was specified and we have a FileIndex, assume + * it is a Bacula created bsr (or the equivalent). We + * then save the bsr where the match occurred so that + * after processing the record or records, we can update + * the found count. I.e. rec->bsr points to the bsr that + * satisfied the match. + */ + if (bsr->count && bsr->FileIndex) { + rec->bsr = bsr; + if (bsr->next && rec->FileIndex != bsr->LastFI) { + bsr->LastFI = rec->FileIndex; + } + Dmsg1(dbglevel, "Leave match_all 1 found=%d\n", bsr->found); + return 1; /* this is a complete match */ + } + + /* + * The selections below are not used by Bacula's + * restore command, and don't work because of + * the rec->bsr = bsr optimization above. + */ + if (sessrec) { + if (!match_jobid(bsr, bsr->JobId, sessrec, 1)) { + Dmsg0(dbglevel, "fail on JobId\n"); + goto no_match; + } + if (!match_job(bsr, bsr->job, sessrec, 1)) { + Dmsg0(dbglevel, "fail on Job\n"); + goto no_match; + } + if (!match_client(bsr, bsr->client, sessrec, 1)) { + Dmsg0(dbglevel, "fail on Client\n"); + goto no_match; + } + if (!match_job_type(bsr, bsr->JobType, sessrec, 1)) { + Dmsg0(dbglevel, "fail on Job type\n"); + goto no_match; + } + if (!match_job_level(bsr, bsr->JobLevel, sessrec, 1)) { + Dmsg0(dbglevel, "fail on Job level\n"); + goto no_match; + } + if (!match_stream(bsr, bsr->stream, rec, 1)) { + Dmsg0(dbglevel, "fail on stream\n"); + goto no_match; + } + } + return 1; + +no_match: + if (bsr->count && bsr->found >= bsr->count) { + bsr->done = true; + if (bsr->next) bsr->root->cur_bsr = bsr->next; + Dmsg1(dbglevel, "bsr done: Volume=%s\n", bsr->volume->VolumeName); + } + if (bsr->next) { + done = bsr->done && done; + bsr = bsr->next; + continue; + } + if (bsr->done && done) { + Dmsg0(dbglevel, "Leave match all -1\n"); + return -1; + } + Dmsg1(dbglevel, "Leave match all 0, repos=%d\n", bsr->reposition); + return 0; + } + return 0; +} + /* * Match all the components of current record * returns 1 on match @@ -380,7 +497,6 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, { Dmsg0(dbglevel, "Enter match_all\n"); if (bsr->done) { -// Dmsg0(dbglevel, "bsr->done set\n"); goto no_match; } if (!match_volume(bsr, bsr->volume, volrec, 1)) { @@ -391,18 +507,11 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, Dmsg2(dbglevel, "OK bsr match bsr_vol=%s read_vol=%s\n", bsr->volume->VolumeName, volrec->VolumeName); - if (!match_volfile(bsr, bsr->volfile, rec, 1)) { - if (bsr->volfile) { - Dmsg3(dbglevel, "Fail on file=%u. bsr=%u,%u\n", - rec->File, bsr->volfile->sfile, bsr->volfile->efile); - } - goto no_match; - } - if (!match_voladdr(bsr, bsr->voladdr, rec, 1)) { if (bsr->voladdr) { Dmsg3(dbglevel, "Fail on Addr=%llu. bsr=%llu,%llu\n", get_record_address(rec), bsr->voladdr->saddr, bsr->voladdr->eaddr); + dump_record(rec); } goto no_match; } @@ -421,7 +530,7 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, } /* NOTE!! This test MUST come after sesstime and sessid tests */ - if (!match_findex(bsr, bsr->FileIndex, rec, 1)) { + if (!match_findex(bsr, rec, 1)) { Dmsg3(dbglevel, "Fail on findex=%d. bsr=%d,%d\n", rec->FileIndex, bsr->FileIndex->findex, bsr->FileIndex->findex2); goto no_match; @@ -506,7 +615,7 @@ static int match_volume(BSR *bsr, BSR_VOLUME *volume, VOLUME_LABEL *volrec, bool return 0; /* Volume must match */ } if (strcmp(volume->VolumeName, volrec->VolumeName) == 0) { - Dmsg1(dbglevel, "match_volume=%s\n", volrec->VolumeName); + Dmsg1(dbglevel, "OK match_volume=%s\n", volrec->VolumeName); return 1; } if (volume->next) { @@ -585,74 +694,27 @@ static int match_jobid(BSR *bsr, BSR_JOBID *jobid, SESSION_LABEL *sessrec, bool return 0; } -static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, bool done) -{ - if (!volfile) { - return 1; /* no specification matches all */ - } -/* - * The following code is turned off because this should now work - * with disk files too, though since a "volfile" is 4GB, it does - * not improve performance much. - */ -#ifdef xxx - /* For the moment, these tests work only with tapes. */ - if (!(rec->state & REC_ISTAPE)) { - return 1; /* All File records OK for this match */ - } - Dmsg3(dbglevel, "match_volfile: sfile=%u efile=%u recfile=%u\n", - volfile->sfile, volfile->efile, rec->File); -#endif - if (volfile->sfile <= rec->File && volfile->efile >= rec->File) { - return 1; - } - /* Once we get past last efile, we are done */ - if (rec->File > volfile->efile) { - volfile->done = true; /* set local done */ - } - if (volfile->next) { - return match_volfile(bsr, volfile->next, rec, volfile->done && done); - } - - /* If we are done and all prior matches are done, this bsr is finished */ - if (volfile->done && done) { - bsr->done = true; - bsr->root->reposition = true; - Dmsg2(dbglevel, "bsr done from volfile rec=%u volefile=%u\n", - rec->File, volfile->efile); - } - return 0; -} - static int match_voladdr(BSR *bsr, BSR_VOLADDR *voladdr, DEV_RECORD *rec, bool done) { if (!voladdr) { return 1; /* no specification matches all */ } -#ifdef xxx - - /* For the moment, these tests work only with disk. */ - if (rec->state & REC_ISTAPE) { - uint32_t sFile = (voladdr->saddr)>>32; - uint32_t eFile = (voladdr->eaddr)>>32; - if (sFile <= rec->File && eFile >= rec->File) { - return 1; - } - } - -#endif - uint64_t addr = get_record_address(rec); Dmsg6(dbglevel, "match_voladdr: saddr=%llu eaddr=%llu recaddr=%llu sfile=%u efile=%u recfile=%u\n", - voladdr->saddr, voladdr->eaddr, addr, voladdr->saddr>>32, voladdr->eaddr>>32, addr>>32); + voladdr->saddr, voladdr->eaddr, addr, (uint32_t)(voladdr->saddr>>32), + (uint32_t)(voladdr->eaddr>>32), (uint32_t)(addr>>32)); if (voladdr->saddr <= addr && voladdr->eaddr >= addr) { + Dmsg1(dbglevel, "OK match voladdr=%lld\n", addr); return 1; } /* Once we get past last eblock, we are done */ if (addr > voladdr->eaddr) { voladdr->done = true; /* set local done */ + if (!voladdr->next) { /* done with everything? */ + bsr->done = true; /* yes */ + } } if (voladdr->next) { return match_voladdr(bsr, voladdr->next, rec, voladdr->done && done); @@ -728,57 +790,51 @@ static int match_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_RECORD *rec) * When reading the Volume, the Volume Findex (rec->FileIndex) always * are found in sequential order. Thus we can make optimizations. * - * ***FIXME*** optimizations - * We could optimize by removing the recursion. */ -static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, bool done) +static int match_findex(BSR *bsr, DEV_RECORD *rec, bool done) { + BSR_FINDEX *findex = bsr->FileIndex; + BSR_FINDEX *next; + if (!findex) { return 1; /* no specification matches all */ } - if (!findex->done) { + + for ( ;; ) { if (findex->findex <= rec->FileIndex && findex->findex2 >= rec->FileIndex) { - Dmsg3(dbglevel, "Match on findex=%d. bsrFIs=%d,%d\n", + Dmsg3(dbglevel, "Match on recFindex=%d. bsrFIs=%d,%d\n", rec->FileIndex, findex->findex, findex->findex2); return 1; } if (rec->FileIndex > findex->findex2) { - findex->done = true; + /* TODO: See if we really want to modify the findex when we will try + * to seek backward */ + if (findex->next) { + next = findex->next; + Dmsg3(dbglevel, "No match recFindex=%d. bsrFIs=%d,%d\n", + rec->FileIndex, findex->findex, findex->findex2); + free(findex); + findex = next; + bsr->FileIndex = findex; + continue; + } else { + bsr->done = true; + bsr->root->reposition = true; + } } + return 0; } - if (findex->next) { - return match_findex(bsr, findex->next, rec, findex->done && done); - } - if (findex->done && done) { - bsr->done = true; - bsr->root->reposition = true; - Dmsg1(dbglevel, "bsr done from findex %d\n", rec->FileIndex); - } - return 0; } -uint64_t get_bsr_start_addr(BSR *bsr, uint32_t *file, uint32_t *block) +uint64_t get_bsr_start_addr(BSR *bsr) { uint64_t bsr_addr = 0; - uint32_t sfile = 0, sblock = 0; if (bsr) { if (bsr->voladdr) { bsr_addr = bsr->voladdr->saddr; - sfile = bsr_addr>>32; - sblock = (uint32_t)bsr_addr; - - } else if (bsr->volfile && bsr->volblock) { - bsr_addr = (((uint64_t)bsr->volfile->sfile)<<32)|bsr->volblock->sblock; - sfile = bsr->volfile->sfile; - sblock = bsr->volblock->sblock; } } - if (file && block) { - *file = sfile; - *block = sblock; - } - return bsr_addr; } diff --git a/bacula/src/stored/mount.c b/bacula/src/stored/mount.c index 960ef83cdf..e763464f2b 100644 --- a/bacula/src/stored/mount.c +++ b/bacula/src/stored/mount.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -63,10 +63,10 @@ bool DCR::mount_next_write_volume() { int retry = 0; bool ask = false, recycle, autochanger; - int mode; DCR *dcr = this; Enter(200); + set_ameta(); Dmsg2(100, "Enter mount_next_volume(release=%d) dev=%s\n", dev->must_unload(), dev->print_name()); @@ -177,40 +177,28 @@ mount_next_vol: dev->VolHdr.VolumeName, dev->print_name()); if (dev->poll && dev->has_cap(CAP_CLOSEONPOLL)) { - dev->close(); + dev->close(this); free_volume(dev); } - /* Ensure the device is open */ - if (dev->has_cap(CAP_STREAM)) { - mode = OPEN_WRITE_ONLY; - } else { - mode = OPEN_READ_WRITE; - } /* Try autolabel if enabled */ Dmsg1(100, "Try open Vol=%s\n", getVolCatName()); - if (!dev->open(dcr, mode)) { + if (!dev->open_device(dcr, OPEN_READ_WRITE)) { Dmsg1(100, "Try autolabel Vol=%s\n", getVolCatName()); - try_autolabel(false); /* try to create a new volume label */ + if (!dev->poll) { + try_autolabel(false); /* try to create a new volume label */ + } } - while (!dev->open(dcr, mode)) { - Dmsg1(100, "open_device failed: ERR=%s\n", dev->bstrerror()); - if ((dev->is_file() && dev->is_removable()) || dev->is_dvd()) { + while (!dev->open_device(dcr, OPEN_READ_WRITE)) { + Dmsg1(100, "open_device failed: ERR=%s", dev->bstrerror()); + if (dev->is_file() && dev->is_removable()) { bool ok = true; Dmsg0(150, "call scan_dir_for_vol\n"); - if (dev->is_dvd()) { - if (!dev->mount(0)) { - ok = false; - } - } if (ok && dev->scan_dir_for_volume(dcr)) { - if (dev->open(dcr, mode)) { + if (dev->open_device(dcr, OPEN_READ_WRITE)) { break; /* got a valid volume */ } } - if (ok && dev->is_dvd()) { - dev->unmount(0); - } } if (try_autolabel(false) == try_read_vol) { break; /* created a new volume label */ @@ -262,7 +250,7 @@ read_volume: if (!find_a_volume()) { goto mount_next_vol; } - dev->VolCatInfo = VolCatInfo; /* structure assignment */ + dev->set_volcatinfo_from_dcr(this); } /* @@ -281,7 +269,7 @@ read_volume: recycle = strcmp(dev->VolCatInfo.VolCatStatus, "Recycle") == 0; if (dev->VolHdr.LabelType == PRE_LABEL || recycle) { dcr->WroteVol = false; - if (!dcr->rewrite_volume_label(recycle)) { + if (!dev->rewrite_volume_label(dcr, recycle)) { mark_volume_in_error(); goto mount_next_vol; } @@ -305,7 +293,7 @@ read_volume: goto mount_next_vol; } - if (!is_eod_valid()) { + if (!dev->is_eod_valid(dcr)) { Dmsg0(100, "goto mount_next_vol\n"); goto mount_next_vol; } @@ -351,7 +339,7 @@ bool DCR::find_a_volume() /* Do we have a candidate volume? */ if (dev->vol) { bstrncpy(VolumeName, dev->vol->vol_name, sizeof(VolumeName)); - have_vol = dir_get_volume_info(this, GET_VOL_INFO_FOR_WRITE); + have_vol = dir_get_volume_info(this, VolumeName, GET_VOL_INFO_FOR_WRITE); } /* * Get Director's idea of what tape we should have mounted. @@ -389,7 +377,7 @@ bool DCR::find_a_volume() if (dcr->haveVolCatInfo()) { return true; } - return dir_get_volume_info(dcr, GET_VOL_INFO_FOR_WRITE); + return dir_get_volume_info(dcr, VolumeName, GET_VOL_INFO_FOR_WRITE); } int DCR::check_volume_label(bool &ask, bool &autochanger) @@ -398,16 +386,17 @@ int DCR::check_volume_label(bool &ask, bool &autochanger) Enter(200); + set_ameta(); /* * If we are writing to a stream device, ASSUME the volume label * is correct. */ if (dev->has_cap(CAP_STREAM)) { vol_label_status = VOL_OK; - create_volume_header(dev, VolumeName, "Default", false /* not DVD */); + create_volume_header(dev, VolumeName, "Default", false); dev->VolHdr.LabelType = PRE_LABEL; } else { - vol_label_status = read_dev_volume_label(this); + vol_label_status = dev->read_dev_volume_label(this); } if (job_canceled(jcr)) { goto check_bail_out; @@ -458,13 +447,13 @@ int DCR::check_volume_label(bool &ask, bool &autochanger) /* Check if this is a valid Volume in the pool */ bstrncpy(saveVolumeName, VolumeName, sizeof(saveVolumeName)); bstrncpy(VolumeName, dev->VolHdr.VolumeName, sizeof(VolumeName)); - if (!dir_get_volume_info(this, GET_VOL_INFO_FOR_WRITE)) { + if (!dir_get_volume_info(this, VolumeName, GET_VOL_INFO_FOR_WRITE)) { POOL_MEM vol_info_msg; pm_strcpy(vol_info_msg, jcr->dir_bsock->msg); /* save error message */ /* Restore desired volume name, note device info out of sync */ /* This gets the info regardless of the Pool */ bstrncpy(VolumeName, dev->VolHdr.VolumeName, sizeof(VolumeName)); - if (autochanger && !dir_get_volume_info(this, GET_VOL_INFO_FOR_READ)) { + if (autochanger && !dir_get_volume_info(this, VolumeName, GET_VOL_INFO_FOR_READ)) { /* * If we get here, we know we cannot write on the Volume, * and we know that we cannot read it either, so it @@ -509,11 +498,6 @@ int DCR::check_volume_label(bool &ask, bool &autochanger) * At this point, we assume we have a blank tape mounted. */ case VOL_IO_ERROR: - if (dev->is_dvd()) { - Jmsg(jcr, M_FATAL, 0, "%s", jcr->errmsg); - mark_volume_in_error(); - goto check_bail_out; /* we could not write on DVD */ - } /* Fall through wanted */ case VOL_NO_LABEL: switch (try_autolabel(true)) { @@ -538,7 +522,7 @@ int DCR::check_volume_label(bool &ask, bool &autochanger) ask = true; /* Needed, so the medium can be changed */ if (dev->requires_mount()) { - dev->close(); + dev->close(this); free_volume(dev); } goto check_next_volume; @@ -572,7 +556,7 @@ bool DCR::is_suitable_volume_mounted() return false; /* no */ } bstrncpy(VolumeName, dev->VolHdr.VolumeName, sizeof(VolumeName)); - ok = dir_get_volume_info(this, GET_VOL_INFO_FOR_WRITE); + ok = dir_get_volume_info(this, VolumeName, GET_VOL_INFO_FOR_WRITE); if (!ok) { Dmsg1(40, "dir_get_volume_info failed: %s", jcr->errmsg); dev->set_wait(); @@ -642,99 +626,6 @@ void DCR::do_swapping(bool is_writing) } } - -/* - * Check if the current position on the volume corresponds to - * what is in the catalog. - */ -bool DCR::is_eod_valid() -{ - if (dev->is_dvd()) { - char ed1[50], ed2[50]; - if (dev->VolCatInfo.VolCatBytes == dev->part_start + dev->part_size) { - Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volume \"%s\"" - " part=%d size=%s\n"), VolumeName, - dev->part, edit_uint64_with_commas(dev->VolCatInfo.VolCatBytes,ed1)); - } else { - Jmsg(jcr, M_ERROR, 0, _("Bacula cannot write on DVD Volume \"%s\" because: " - "The sizes do not match! Volume=%s Catalog=%s\n"), - VolumeName, - edit_uint64_with_commas(dev->part_start + dev->part_size, ed1), - edit_uint64_with_commas(dev->VolCatInfo.VolCatBytes, ed2)); - mark_volume_in_error(); - return false; - } - } else if (dev->is_tape()) { - /* - * Check if we are positioned on the tape at the same place - * that the database says we should be. - */ - if (dev->VolCatInfo.VolCatFiles == dev->get_file()) { - Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volume \"%s\" at file=%d.\n"), - VolumeName, dev->get_file()); - } else if (dev->get_file() > dev->VolCatInfo.VolCatFiles) { - Jmsg(jcr, M_WARNING, 0, _("For Volume \"%s\":\n" - "The number of files mismatch! Volume=%u Catalog=%u\n" - "Correcting Catalog\n"), - VolumeName, dev->get_file(), dev->VolCatInfo.VolCatFiles); - dev->VolCatInfo.VolCatFiles = dev->get_file(); - dev->VolCatInfo.VolCatBlocks = dev->get_block_num(); - if (!dir_update_volume_info(this, false, true)) { - Jmsg(jcr, M_WARNING, 0, _("Error updating Catalog\n")); - mark_volume_in_error(); - return false; - } - } else { - Jmsg(jcr, M_ERROR, 0, _("Bacula cannot write on tape Volume \"%s\" because:\n" - "The number of files mismatch! Volume=%u Catalog=%u\n"), - VolumeName, dev->get_file(), dev->VolCatInfo.VolCatFiles); - mark_volume_in_error(); - return false; - } - /* - * File device. - */ - } else if (dev->is_file()) { - char ed1[50], ed2[50]; - boffset_t pos; - - pos = dev->lseek(this, (boffset_t)0, SEEK_END); - if (dev->VolCatInfo.VolCatAmetaBytes == (uint64_t)pos) { - Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volume \"%s\"" - " size=%s\n"), VolumeName, - edit_uint64_with_commas(dev->VolCatInfo.VolCatAmetaBytes, ed1)); - } else if ((uint64_t)pos >= dev->VolCatInfo.VolCatAmetaBytes) { - if ((uint64_t)pos != dev->VolCatInfo.VolCatAmetaBytes) { - Jmsg(jcr, M_WARNING, 0, _("For Volume \"%s\":\n" - " The sizes do not match! Volume=%s Catalog=%s\n" - " Correcting Catalog\n"), - VolumeName, edit_uint64_with_commas(pos, ed1), - edit_uint64_with_commas(dev->VolCatInfo.VolCatAmetaBytes, ed2)); - } - dev->VolCatInfo.VolCatAmetaBytes = pos; - dev->VolCatInfo.VolCatBytes = pos; - dev->VolCatInfo.VolCatFiles = (uint32_t)(pos >> 32); - if (!dir_update_volume_info(this, false, true)) { - Jmsg(jcr, M_WARNING, 0, _("Error updating Catalog\n")); - mark_volume_in_error(); - return false; - } - } else { - Mmsg(jcr->errmsg, _("Bacula cannot write on disk Volume \"%s\" because: " - "The sizes do not match! Volume=%s Catalog=%s\n"), - VolumeName, - edit_uint64_with_commas(pos, ed1), - edit_uint64_with_commas(dev->VolCatInfo.VolCatBytes, ed2)); - Jmsg(jcr, M_ERROR, 0, jcr->errmsg); - Dmsg0(100, jcr->errmsg); - mark_volume_in_error(); - return false; - } - } - return true; -} - - /* * If permitted, we label the device, make sure we can do * it by checking that the VolCatBytes is zero => not labeled, @@ -769,8 +660,8 @@ int DCR::try_autolabel(bool opened) "Recycle") == 0))) { Dmsg1(40, "Create new volume label vol=%s\n", VolumeName); /* Create a new Volume label and write it to the device */ - if (!write_new_volume_label_to_dev(dcr, VolumeName, - pool_name, false, /* no relabel */ false /* defer DVD label */)) { + if (!dev->write_volume_label(dcr, VolumeName, + pool_name, false, /* no relabel */ false /* defer label */)) { Dmsg2(100, "write_vol_label failed. vol=%s, pool=%s\n", VolumeName, pool_name); if (opened) { @@ -863,12 +754,12 @@ void DCR::release_volume() if (dev->is_open() && (!dev->is_tape() || !dev->has_cap(CAP_ALWAYSOPEN))) { generate_plugin_event(jcr, bsdEventDeviceClose, this); - dev->close(); + dev->close(this); } /* If we have not closed the device, then at least rewind the tape */ if (dev->is_open()) { - dev->offline_or_rewind(); + dev->offline_or_rewind(this); } /* @@ -943,7 +834,7 @@ bool mount_next_read_volume(DCR *dcr) */ if (jcr->NumReadVolumes > 1 && jcr->CurReadVolume < jcr->NumReadVolumes) { dev->Lock(); - dev->close(); + dev->close(dcr); dev->set_read(); dcr->set_reserved_for_read(); dev->Unlock(); diff --git a/bacula/src/stored/null_dev.c b/bacula/src/stored/null_dev.c new file mode 100644 index 0000000000..3703b7ee4f --- /dev/null +++ b/bacula/src/stored/null_dev.c @@ -0,0 +1,33 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * Null driver code + * + * written by, Kern Sibbald, MMXVI + * + */ + +#include "bacula.h" +#include "stored.h" + +const char *null_dev::print_type() +{ + return "Null"; +} diff --git a/bacula/src/stored/null_dev.h b/bacula/src/stored/null_dev.h new file mode 100644 index 0000000000..33b85e78ef --- /dev/null +++ b/bacula/src/stored/null_dev.h @@ -0,0 +1,34 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * NULL driver -- normally only used for performance testing + */ + +#ifndef __NULL_DEV_ +#define __NULL_DEV_ + +class null_dev : public file_dev { +public: + + null_dev() { }; + ~null_dev() { }; + const char *print_type(); +}; + +#endif /* __NULL_DEV_ */ diff --git a/bacula/src/stored/os.c b/bacula/src/stored/os.c index 2f754eb25b..0d181e522f 100644 --- a/bacula/src/stored/os.c +++ b/bacula/src/stored/os.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. diff --git a/bacula/src/stored/parse_bsr.c b/bacula/src/stored/parse_bsr.c index 049f41254a..87625e2dc4 100644 --- a/bacula/src/stored/parse_bsr.c +++ b/bacula/src/stored/parse_bsr.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -28,7 +28,7 @@ #include "stored.h" static void s_err(const char *file, int line, LEX *lc, const char *msg, ...); -static bool add_restore_volume(JCR *jcr, VOL_LIST *vol); +static bool add_restore_volume(JCR *jcr, VOL_LIST *vol, bool add_to_read_list); typedef BSR * (ITEM_HANDLER)(LEX *lc, BSR *bsr); @@ -92,7 +92,7 @@ struct kw_items items[] = { /* * Create a BSR record */ -static BSR *new_bsr() +BSR *new_bsr() { BSR *bsr = (BSR *)malloc(sizeof(BSR)); memset(bsr, 0, sizeof(BSR)); @@ -720,11 +720,18 @@ void dump_volblock(BSR_VOLBLOCK *volblock) } } -void dump_voladdr(BSR_VOLADDR *voladdr) +void dump_voladdr(DEVICE *dev, BSR_VOLADDR *voladdr) { if (voladdr) { - Pmsg2(-1, _("VolAddr : %llu-%llu\n"), voladdr->saddr, voladdr->eaddr); - dump_voladdr(voladdr->next); + if (dev) { + char ed1[50], ed2[50]; + Pmsg2(-1, _("VolAddr : %s-%llu\n"), + dev->print_addr(ed1, sizeof(ed1), voladdr->saddr), + dev->print_addr(ed2, sizeof(ed2), voladdr->eaddr)); + } else { + Pmsg2(-1, _("VolAddr : %llu-%llu\n"), voladdr->saddr, voladdr->eaddr); + } + dump_voladdr(dev, voladdr->next); } } @@ -801,7 +808,7 @@ void dump_sesstime(BSR_SESSTIME *sesstime) } -void dump_bsr(BSR *bsr, bool recurse) +void dump_bsr(DEVICE *dev, BSR *bsr, bool recurse) { int64_t save_debug = debug_level; debug_level = 1; @@ -817,7 +824,7 @@ void dump_bsr(BSR *bsr, bool recurse) dump_sesstime(bsr->sesstime); dump_volfile(bsr->volfile); dump_volblock(bsr->volblock); - dump_voladdr(bsr->voladdr); + dump_voladdr(dev, bsr->voladdr); dump_client(bsr->client); dump_jobid(bsr->JobId); dump_job(bsr->job); @@ -832,13 +839,11 @@ void dump_bsr(BSR *bsr, bool recurse) Pmsg1(-1, _("fast_reject : %d\n"), bsr->use_fast_rejection); if (recurse && bsr->next) { Pmsg0(-1, "\n"); - dump_bsr(bsr->next, true); + dump_bsr(dev, bsr->next, true); } debug_level = save_debug; } - - /********************************************************************* * * Free bsr resources @@ -920,7 +925,7 @@ static VOL_LIST *new_restore_volume() * Create a list of Volumes (and Slots and Start positions) to be * used in the current restore job. */ -void create_restore_volume_list(JCR *jcr) +void create_restore_volume_list(JCR *jcr, bool add_to_read_list) { char *p, *n; VOL_LIST *vol; @@ -954,7 +959,7 @@ void create_restore_volume_list(JCR *jcr) bstrncpy(vol->device, bsrvol->device, sizeof(vol->device)); vol->Slot = bsrvol->Slot; vol->start_file = sfile; - if (add_restore_volume(jcr, vol)) { + if (add_restore_volume(jcr, vol, add_to_read_list)) { jcr->NumReadVolumes++; Dmsg2(400, "Added volume=%s mediatype=%s\n", vol->VolumeName, vol->MediaType); @@ -975,7 +980,7 @@ void create_restore_volume_list(JCR *jcr) vol = new_restore_volume(); bstrncpy(vol->VolumeName, p, sizeof(vol->VolumeName)); bstrncpy(vol->MediaType, jcr->dcr->media_type, sizeof(vol->MediaType)); - if (add_restore_volume(jcr, vol)) { + if (add_restore_volume(jcr, vol, add_to_read_list)) { jcr->NumReadVolumes++; } else { free((char *)vol); @@ -992,12 +997,14 @@ void create_restore_volume_list(JCR *jcr) * returns: 1 if volume added * 0 if volume already in list */ -static bool add_restore_volume(JCR *jcr, VOL_LIST *vol) +static bool add_restore_volume(JCR *jcr, VOL_LIST *vol, bool add_to_read_list) { VOL_LIST *next = jcr->VolList; - /* Add volume to volume manager's read list */ - add_read_volume(jcr, vol->VolumeName); + if (add_to_read_list) { + /* Add volume to volume manager's read list */ + add_read_volume(jcr, vol->VolumeName); + } if (!next) { /* list empty ? */ jcr->VolList = vol; /* yes, add volume */ diff --git a/bacula/src/stored/protos.h b/bacula/src/stored/protos.h index adfdbcfed5..cb9ab55d3a 100644 --- a/bacula/src/stored/protos.h +++ b/bacula/src/stored/protos.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -41,9 +41,10 @@ enum get_vol_info_rw { GET_VOL_INFO_FOR_WRITE, GET_VOL_INFO_FOR_READ }; -bool dir_get_volume_info(DCR *dcr, enum get_vol_info_rw); +bool dir_get_volume_info(DCR *dcr, const char *VolumeName, enum get_vol_info_rw); bool dir_find_next_appendable_volume(DCR *dcr); -bool dir_update_volume_info(DCR *dcr, bool label, bool update_LastWritten); +bool dir_update_volume_info(DCR *dcr, bool label, bool update_LastWritten, + bool use_dcr=false); bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr); bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool read_access); bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec); @@ -54,6 +55,37 @@ bool flush_jobmedia_queue(JCR *jcr); bool dir_update_device(JCR *jcr, DEVICE *dev); bool dir_update_changer(JCR *jcr, AUTOCHANGER *changer); +/* class AskDirHandler allows btool's utilities to overwrite the functions above + * and even to overwrite the functions in their own sub-class like in btape + */ +class AskDirHandler +{ + /* this class should be an abstract class */ +public: + AskDirHandler() {} + virtual ~AskDirHandler() {} + virtual bool dir_find_next_appendable_volume(DCR *dcr) { return 1;} + virtual bool dir_update_volume_info(DCR *dcr, bool relabel, bool update_LastWritten, bool use_dcr) { return 1; } + virtual bool dir_create_jobmedia_record(DCR *dcr, bool zero) { return 1; } + virtual bool flush_jobmedia_queue(JCR *jcr) { return true; } + virtual bool dir_ask_sysop_to_create_appendable_volume(DCR *dcr) { return 1; } + virtual bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec) { return 1;} + virtual bool dir_send_job_status(JCR *jcr) {return 1;} + virtual bool dir_ask_sysop_to_mount_volume(DCR *dcr, bool writing); + virtual bool dir_get_volume_info(DCR *dcr, const char *VolumeName, enum get_vol_info_rw writing); +}; + +class BtoolsAskDirHandler: public AskDirHandler +{ + /* if AskDirHandler was an abstract class, implement the method here */ +public: + BtoolsAskDirHandler() {} + virtual ~BtoolsAskDirHandler() {} +}; + +/* activate the functions provided by the btool's handler */ +AskDirHandler *init_askdir_handler(AskDirHandler *new_askdir_handler); + /* authenticate.c */ bool authenticate_director(JCR *jcr); int authenticate_filed(JCR *jcr, BSOCK *fd, int FDVersion); @@ -69,8 +101,7 @@ char *edit_device_codes(DCR *dcr, char *omsg, const char *imsg, const char *c int get_autochanger_loaded_slot(DCR *dcr); /* From block.c */ -void dump_block(DEV_BLOCK *b, const char *msg); -DEV_BLOCK *new_block(DEVICE *dev); +void dump_block(DEVICE *dev, DEV_BLOCK *b, const char *msg, bool force=false); DEV_BLOCK *dup_block(DEV_BLOCK *eblock); void init_block_write(DEV_BLOCK *block); void empty_block(DEV_BLOCK *block); @@ -82,8 +113,11 @@ bool terminate_writing_volume(DCR *dcr); /* From block_util.c */ bool terminate_writing_volume(DCR *dcr); -bool user_volume_size_reached(DCR *dcr, bool quiet); +bool is_user_volume_size_reached(DCR *dcr, bool quiet); bool check_for_newvol_or_newfile(DCR *dcr); +bool do_new_file_bookkeeping(DCR *dcr); +void reread_last_block(DCR *dcr); + /* From butil.c -- utilities for SD tool programs */ void setup_me(); @@ -94,7 +128,8 @@ void display_tape_error_status(JCR *jcr, DEVICE *dev); /* From dev.c */ -DEVICE *init_dev(JCR *jcr, DEVRES *device); +void sd_list_loaded_drivers(alist *list); +DEVICE *init_dev(JCR *jcr, DEVRES *device, bool adata=false); bool can_open_mounted_dev(DEVICE *dev); bool load_dev(DEVICE *dev); int write_block(DEVICE *dev); @@ -106,20 +141,7 @@ void init_device_wait_timers(DCR *dcr); void init_jcr_device_wait_timers(JCR *jcr); bool double_dev_wait_time(DEVICE *dev); -/* From dvd.c */ -int dvd_open_next_part(DCR *dcr); -bool dvd_write_part(DCR *dcr); -bool dvd_close_job(DCR *dcr); -void make_mounted_dvd_filename(DEVICE *dev, POOL_MEM &archive_name); -void make_spooled_dvd_filename(DEVICE *dev, POOL_MEM &archive_name); -bool truncate_dvd(DCR *dcr); -bool check_can_write_on_non_blank_dvd(DCR *dcr); -int find_num_dvd_parts(DCR *dcr); -boffset_t lseek_dvd(DCR *dcr, boffset_t offset, int whence); -void dvd_remove_empty_part(DCR *dcr); - /* From device.c */ -bool open_device(DCR *dcr); bool first_open_device(DCR *dcr); bool fixup_device_block_write_error(DCR *dcr, int retries=4); void set_start_vol_position(DCR *dcr); @@ -128,7 +150,7 @@ void set_new_file_parameters(DCR *dcr); /* From dircmd.c */ void *handle_connection_request(void *arg); - +extern int use_new_match_all; /* Temp variable, to be removed when the new code is tested */ /* From fd_cmds.c */ void run_job(JCR *jcr); @@ -150,22 +172,17 @@ bool is_client_connection(BSOCK *bs); void handle_client_connection(BSOCK *fd); /* From label.c */ -int read_dev_volume_label(DCR *dcr); -int read_dvd_volume_label(DCR *dcr, bool write); void create_session_label(DCR *dcr, DEV_RECORD *rec, int label); -void create_volume_header(DEVICE *dev, const char *VolName, const char *PoolName, bool dvdnow); +void create_volume_header(DEVICE *dev, const char *VolName, const char *PoolName, bool no_prelabel); #define ANSI_VOL_LABEL 0 #define ANSI_EOF_LABEL 1 #define ANSI_EOV_LABEL 2 bool write_ansi_ibm_labels(DCR *dcr, int type, const char *VolName); int read_ansi_ibm_label(DCR *dcr); bool write_session_label(DCR *dcr, int label); -void dump_volume_label(DEVICE *dev); int dump_label_record(DEVICE *dev, DEV_RECORD *rec, int verbose, bool check_err); bool unser_volume_label(DEVICE *dev, DEV_RECORD *rec); bool unser_session_label(SESSION_LABEL *label, DEV_RECORD *rec); -bool write_new_volume_label_to_dev(DCR *dcr, const char *VolName, - const char *PoolName, bool relabel, bool dvdnow); /* From locks.c */ void _lock_device(const char *file, int line, DEVICE *dev); @@ -182,25 +199,24 @@ int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, int match_bsr_block(BSR *bsr, DEV_BLOCK *block); void position_bsr_block(BSR *bsr, DEV_BLOCK *block); BSR *find_next_bsr(BSR *root_bsr, DEVICE *dev); -bool is_this_bsr_done(BSR *bsr, DEV_RECORD *rec); -uint64_t get_bsr_start_addr(BSR *bsr, - uint32_t *file=NULL, - uint32_t *block=NULL); - +bool is_this_bsr_done(JCR *jcr, BSR *bsr, DEV_RECORD *rec); +uint64_t get_bsr_start_addr(BSR *bsr); /* From mount.c */ bool mount_next_read_volume(DCR *dcr); /* From parse_bsr.c */ +BSR *new_bsr(); BSR *parse_bsr(JCR *jcr, char *lf); -void dump_bsr(BSR *bsr, bool recurse); +void dump_bsr(DEVICE *dev, BSR *bsr, bool recurse); void free_bsr(BSR *bsr); void free_restore_volume_list(JCR *jcr); -void create_restore_volume_list(JCR *jcr); +void create_restore_volume_list(JCR *jcr, bool add_to_read_list); /* From record.c */ const char *FI_to_ascii(char *buf, int fi); const char *stream_to_ascii(char *buf, int stream, int fi); +const char *stream_to_ascii_ex(char *buf, int stream, int fi); bool write_record_to_block(DCR *dcr, DEV_RECORD *rec); bool can_write_record_to_block(DEV_BLOCK *block, DEV_RECORD *rec); bool read_record_from_block(DCR *dcr, DEV_RECORD *rec); @@ -208,8 +224,12 @@ DEV_RECORD *new_record(); void free_record(DEV_RECORD *rec); void empty_record(DEV_RECORD *rec); uint64_t get_record_address(DEV_RECORD *rec); +uint64_t get_record_start_address(DEV_RECORD *rec); bool flush_adata_to_device(DCR *dcr); +/* From record_util.c */ +void dump_record(DEV_RECORD *rec); + /* From read_record.c */ bool read_records(DCR *dcr, bool record_cb(DCR *dcr, DEV_RECORD *rec), @@ -259,11 +279,21 @@ bool commit_attribute_spool (JCR *jcr); bool write_block_to_spool_file (DCR *dcr); void list_spool_stats (void sendit(const char *msg, int len, void *sarg), void *arg); +/* From tape_alert.c */ +extern void alert_callback(void *ctx, const char *short_msg, + const char *long_msg, char *Volume, + int severity, int flags, int alertno, utime_t alert_time); + /* From wait.c */ int wait_for_sysop(DCR *dcr); bool wait_for_any_device(JCR *jcr, int &retries); bool wait_for_device(DCR *dcr, int &retries); /* stored_conf.c */ +void store_protocol(LEX *lc, RES_ITEM *item, int index, int pass); +void store_uri_style(LEX *lc, RES_ITEM *item, int index, int pass); +void store_truncate(LEX *lc, RES_ITEM *item, int index, int pass); +void store_upload(LEX *lc, RES_ITEM *item, int index, int pass); void store_devtype(LEX *lc, RES_ITEM *item, int index, int pass); +void store_cloud_driver(LEX *lc, RES_ITEM *item, int index, int pass); void store_maxblocksize(LEX *lc, RES_ITEM *item, int index, int pass); diff --git a/bacula/src/stored/read.c b/bacula/src/stored/read.c index a20db857f0..df3f13d2d9 100644 --- a/bacula/src/stored/read.c +++ b/bacula/src/stored/read.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -67,10 +67,10 @@ bool do_read_data(JCR *jcr) fd->fsend(FD_error); return false; } + dcr->dev->start_of_job(dcr); /* Tell File daemon we will send data */ if (!jcr->is_ok_data_sent) { - /* OK_DATA can have been already sent for copy/migrate by run_job() to avoid dead lock*/ fd->fsend(OK_data); jcr->is_ok_data_sent = true; } @@ -130,6 +130,7 @@ static bool read_record_cb(DCR *dcr, DEV_RECORD *rec) stream_to_ascii(ec2, rec->Stream, rec->FileIndex), wsize); + Dmsg2(640, ">filed: send header stream=0x%lx len=%ld\n", rec->Stream, wsize); /* Send record header to File daemon */ if (!fd->fsend(rec_header, rec->VolSessionId, rec->VolSessionTime, rec->FileIndex, rec->Stream, wsize)) { @@ -158,17 +159,8 @@ static bool read_record_cb(DCR *dcr, DEV_RECORD *rec) } } - /* Debug code: check if we must hangup */ - if (get_hangup() > 0 && (jcr->JobFiles > (uint32_t)get_hangup())) { - jcr->setJobStatus(JS_Incomplete); - Jmsg1(jcr, M_FATAL, 0, "Debug hangup requested after %d files.\n", - get_hangup()); - set_hangup(0); - return false; - } - if (get_blowup() > 0 && (jcr->JobFiles > (uint32_t)get_blowup())) { - Jmsg1(jcr, M_ABORT, 0, "Debug blowup() requested after %d files.\n", - get_blowup()); + /* Debug code: check if we must hangup or blowup */ + if (handle_hangup_blowup(jcr, jcr->JobFiles, jcr->JobBytes)) { return false; } @@ -177,6 +169,7 @@ static bool read_record_cb(DCR *dcr, DEV_RECORD *rec) fd->msglen = wsize; /* Send data record to File daemon */ jcr->JobBytes += wsize; /* increment bytes this job */ + Dmsg1(640, ">filed: send %d bytes data.\n", fd->msglen); if (!fd->send()) { Pmsg1(000, _("Error sending to FD. ERR=%s\n"), fd->bstrerror()); Jmsg1(jcr, M_FATAL, 0, _("Error sending data to Client. ERR=%s\n"), diff --git a/bacula/src/stored/read_records.c b/bacula/src/stored/read_records.c index b42b1810b5..72bb8b5df6 100644 --- a/bacula/src/stored/read_records.c +++ b/bacula/src/stored/read_records.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -35,15 +35,65 @@ #include "stored.h" /* Forward referenced functions */ +static BSR *position_to_first_file(JCR *jcr, DCR *dcr, BSR *bsr); static void handle_session_record(DEVICE *dev, DEV_RECORD *rec, SESSION_LABEL *sessrec); -static BSR *position_to_first_file(JCR *jcr, DCR *dcr); static bool try_repositioning(JCR *jcr, DEV_RECORD *rec, DCR *dcr); #ifdef DEBUG static char *rec_state_bits_to_str(DEV_RECORD *rec); #endif -static const int dbglvl = 190; +static const int dbglvl = 150; static const int no_FileIndex = -999999; +static bool mount_next_vol(JCR *jcr, DCR *dcr, BSR *bsr, + SESSION_LABEL *sessrec, bool *should_stop, + bool record_cb(DCR *dcr, DEV_RECORD *rec), + bool mount_cb(DCR *dcr)) +{ + bool ok = true; + DEVICE *dev = dcr->dev; + *should_stop = false; + + /* We need an other volume */ + volume_unused(dcr); /* mark volume unused */ + if (!mount_cb(dcr)) { + *should_stop = true; + /* + * Create EOT Label so that Media record may + * be properly updated because this is the last + * tape. + */ + DEV_RECORD *trec = new_record(); + trec->FileIndex = EOT_LABEL; + trec->Addr = dev->get_full_addr(); + ok = record_cb(dcr, trec); + free_record(trec); + if (jcr->mount_next_volume) { + jcr->mount_next_volume = false; + dev->clear_eot(); + } + return ok; + } + jcr->mount_next_volume = false; + /* + * The Device can change at the end of a tape, so refresh it + * from the dcr. + */ + dev = dcr->dev; + /* + * We just have a new tape up, now read the label (first record) + * and pass it off to the callback routine, then continue + * most likely reading the previous record. + */ + dcr->read_block_from_device(NO_BLOCK_NUMBER_CHECK); + + DEV_RECORD *trec = new_record(); + read_record_from_block(dcr, trec); + handle_session_record(dev, trec, sessrec); + ok = record_cb(dcr, trec); + free_record(trec); + position_to_first_file(jcr, dcr, bsr); /* We jump to the specified position */ + return ok; +} /* * This subroutine reads all the records and passes them back to your @@ -62,11 +112,16 @@ bool read_records(DCR *dcr, int32_t lastFileIndex; bool ok = true; bool done = false; + bool should_stop; SESSION_LABEL sessrec; dlist *recs; /* linked list of rec packets open */ + char ed1[50]; recs = New(dlist(rec, &rec->link)); - position_to_first_file(jcr, dcr); + /* We go to the first_file unless we need to reposition during an + * interactive restore session (the reposition will be done with a different + * BSR in the for loop */ + position_to_first_file(jcr, dcr, jcr->bsr); jcr->mount_next_volume = false; for ( ; ok && !done; ) { @@ -74,54 +129,29 @@ bool read_records(DCR *dcr, ok = false; break; } - if (!dcr->read_block_from_device(CHECK_BLOCK_NUMBERS)) { + ASSERT2(!dcr->dev->adata, "Called with adata block. Wrong!"); + + if (dev->at_eot() || !dcr->read_block_from_device(CHECK_BLOCK_NUMBERS)) { if (dev->at_eot()) { - DEV_RECORD *trec = new_record(); - Jmsg(jcr, M_INFO, 0, _("End of Volume at file %u on device %s, Volume \"%s\"\n"), - dev->file, dev->print_name(), dcr->VolumeName); - volume_unused(dcr); /* mark volume unused */ - if (!mount_cb(dcr)) { - Jmsg(jcr, M_INFO, 0, _("End of all volumes.\n")); - ok = false; /* Stop everything */ - /* - * Create EOT Label so that Media record may - * be properly updated because this is the last - * tape. - */ - trec->FileIndex = EOT_LABEL; - trec->File = dev->file; - ok = record_cb(dcr, trec); - free_record(trec); - if (jcr->mount_next_volume) { - jcr->mount_next_volume = false; - dev->clear_eot(); - } - break; - } - jcr->mount_next_volume = false; - /* - * The Device can change at the end of a tape, so refresh it - * and the block from the dcr. - */ + Jmsg(jcr, M_INFO, 0, + _("End of Volume \"%s\" at addr=%s on device %s.\n"), + dcr->VolumeName, + dev->print_addr(ed1, sizeof(ed1), dev->EndAddr), + dev->print_name()); + ok = mount_next_vol(jcr, dcr, jcr->bsr, &sessrec, &should_stop, + record_cb, mount_cb); + /* Might have changed after the mount request */ dev = dcr->dev; block = dcr->block; - /* - * We just have a new tape up, now read the label (first record) - * and pass it off to the callback routine, then continue - * most likely reading the previous record. - */ - dcr->read_block_from_device(NO_BLOCK_NUMBER_CHECK); - read_record_from_block(dcr, trec); - handle_session_record(dev, trec, &sessrec); - ok = record_cb(dcr, trec); - free_record(trec); - position_to_first_file(jcr, dcr); - /* After reading label, we must read first data block */ + if (should_stop) { + break; + } continue; } else if (dev->at_eof()) { - Dmsg3(200, "End of file %u on device %s, Volume \"%s\"\n", - dev->file, dev->print_name(), dcr->VolumeName); + Dmsg3(200, "EOF at addr=%s on device %s, Volume \"%s\"\n", + dev->print_addr(ed1, sizeof(ed1), dev->EndAddr), + dev->print_name(), dcr->VolumeName); continue; } else if (dev->is_short_block()) { Jmsg1(jcr, M_ERROR, 0, "%s", dev->errmsg); @@ -138,7 +168,7 @@ bool read_records(DCR *dcr, break; } } - Dmsg2(dbglvl, "Read new block at pos=%u:%u\n", dev->file, dev->block_num); + Dmsg1(dbglvl, "Read new block at pos=%s\n", dev->print_addr(ed1, sizeof(ed1))); #ifdef if_and_when_FAST_BLOCK_REJECTION_is_working /* this does not stop when file/block are too big */ if (!match_bsr_block(jcr->bsr, block)) { @@ -148,7 +178,6 @@ bool read_records(DCR *dcr, continue; /* skip this record */ } #endif - /* * Get a new record for each Job as defined by * VolSessionId and VolSessionTime @@ -157,6 +186,27 @@ bool read_records(DCR *dcr, foreach_dlist(rec, recs) { if (rec->VolSessionId == block->VolSessionId && rec->VolSessionTime == block->VolSessionTime) { + /* When the previous Block of the current record is not correctly ordered, + * if we concat the previous record to the next one, the result is probably + * incorrect. At least the vacuum command should not use this kind of record + */ + if (rec->remainder) { + if (rec->BlockNumber != (block->BlockNumber - 1) + && + rec->BlockNumber != block->BlockNumber) + { + Dmsg3(0, "invalid: rec=%ld block=%ld state=%s\n", + rec->BlockNumber, block->BlockNumber, rec_state_bits_to_str(rec)); + rec->invalid = true; + /* We can discard the current data if needed. The code is very + * tricky in the read_records loop, so it's better to not + * introduce new subtle errors. + */ + if (dcr->discard_invalid_records) { + empty_record(rec); + } + } + } found = true; break; } @@ -168,10 +218,11 @@ bool read_records(DCR *dcr, rec_state_bits_to_str(rec), block->VolSessionId, block->VolSessionTime); } - Dmsg3(dbglvl, "Before read rec loop. stat=%s blk=%d rem=%d\n", rec_state_bits_to_str(rec), - block->BlockNumber, rec->remainder); + Dmsg4(dbglvl, "Before read rec loop. stat=%s blk=%d rem=%d invalid=%d\n", + rec_state_bits_to_str(rec), block->BlockNumber, rec->remainder, rec->invalid); record = 0; rec->state_bits = 0; + rec->BlockNumber = block->BlockNumber; lastFileIndex = no_FileIndex; Dmsg1(dbglvl, "Block %s empty\n", is_block_marked_empty(rec)?"is":"NOT"); for (rec->state_bits=0; ok && !is_block_marked_empty(rec); ) { @@ -180,9 +231,9 @@ bool read_records(DCR *dcr, block->BlockNumber, rec->remainder); break; } - Dmsg5(dbglvl, "read-OK. state_bits=%s blk=%d rem=%d file:block=%u:%u\n", - rec_state_bits_to_str(rec), block->BlockNumber, rec->remainder, - dev->file, dev->block_num); + Dmsg5(dbglvl, "read-OK. state_bits=%s blk=%d rem=%d volume:addr=%s:%llu\n", + rec_state_bits_to_str(rec), block->BlockNumber, rec->remainder, + NPRT(rec->VolumeName), rec->Addr); /* * At this point, we have at least a record header. * Now decide if we want this record or not, but remember @@ -208,11 +259,17 @@ bool read_records(DCR *dcr, } else { rec->match_stat = 0; } + if (rec->invalid) { + Dmsg5(0, "The record %d in block %ld SI=%ld ST=%ld FI=%ld was marked as invalid\n", + rec->RecNum, rec->BlockNumber, rec->VolSessionId, rec->VolSessionTime, rec->FileIndex); + } + //ASSERTD(!rec->invalid || dcr->discard_invalid_records, "Got an invalid record"); /* * Note, we pass *all* labels to the callback routine. If * he wants to know if they matched the bsr, then he must * check the match_stat in the record */ ok = record_cb(dcr, rec); + rec->invalid = false; /* The record was maybe invalid, but the next one is probably good */ #ifdef xxx /* * If this is the end of the Session (EOS) for this record @@ -240,15 +297,15 @@ bool read_records(DCR *dcr, jcr->bsr->reposition); if (rec->match_stat == -1) { /* no more possible matches */ done = true; /* all items found, stop */ - Dmsg2(dbglvl, "All done=(file:block) %u:%u\n", dev->file, dev->block_num); + Dmsg1(dbglvl, "All done Addr=%s\n", dev->print_addr(ed1, sizeof(ed1))); break; } else if (rec->match_stat == 0) { /* no match */ - Dmsg4(dbglvl, "BSR no match: clear rem=%d FI=%d before set_eof pos %u:%u\n", - rec->remainder, rec->FileIndex, dev->file, dev->block_num); + Dmsg3(dbglvl, "BSR no match: clear rem=%d FI=%d before set_eof pos %s\n", + rec->remainder, rec->FileIndex, dev->print_addr(ed1, sizeof(ed1))); rec->remainder = 0; rec->state_bits &= ~REC_PARTIAL_RECORD; if (try_repositioning(jcr, rec, dcr)) { - break; + break; /* We moved on the volume, read next block */ } continue; /* we don't want record, read next one */ } @@ -265,16 +322,22 @@ bool read_records(DCR *dcr, rec_state_bits_to_str(rec), block->BlockNumber, rec->VolSessionId, rec->VolSessionTime, rec->FileIndex); if (lastFileIndex != no_FileIndex && lastFileIndex != rec->FileIndex) { - if (is_this_bsr_done(jcr->bsr, rec) && try_repositioning(jcr, rec, dcr)) { - Dmsg2(dbglvl, "This bsr done, break pos %u:%u\n", - dev->file, dev->block_num); + if (is_this_bsr_done(jcr, jcr->bsr, rec) && try_repositioning(jcr, rec, dcr)) { + Dmsg1(dbglvl, "This bsr done, break pos %s\n", + dev->print_addr(ed1, sizeof(ed1))); break; } Dmsg2(dbglvl, "==== inside LastIndex=%d FileIndex=%d\n", lastFileIndex, rec->FileIndex); } Dmsg2(dbglvl, "==== LastIndex=%d FileIndex=%d\n", lastFileIndex, rec->FileIndex); lastFileIndex = rec->FileIndex; + if (rec->invalid) { + Dmsg5(0, "The record %d in block %ld SI=%ld ST=%ld FI=%ld was marked as invalid\n", + rec->RecNum, rec->BlockNumber, rec->VolSessionId, rec->VolSessionTime, rec->FileIndex); + } + //ASSERTD(!rec->invalid || dcr->discard_invalid_records, "Got an invalid record"); ok = record_cb(dcr, rec); + rec->invalid = false; /* The record was maybe invalid, but the next one is probably good */ #if 0 /* * If we have a digest stream, we check to see if we have @@ -282,21 +345,20 @@ bool read_records(DCR *dcr, * be turned on. */ if (crypto_digest_stream_type(rec->Stream) != CRYPTO_DIGEST_NONE) { - Dmsg3(dbglvl, "=== Have digest FI=%u before bsr check pos %u:%u\n", rec->FileIndex, - dev->file, dev->block_num); - if (is_this_bsr_done(jcr->bsr, rec) && try_repositioning(jcr, rec, dcr)) { + Dmsg3(dbglvl, "=== Have digest FI=%u before bsr check pos %s\n", rec->FileIndex, + dev->print_addr(ed1, sizeof(ed1)); + if (is_this_bsr_done(jcr, jcr->bsr, rec) && try_repositioning(jcr, rec, dcr)) { Dmsg1(dbglvl, "==== BSR done at FI=%d\n", rec->FileIndex); - Dmsg2(dbglvl, "This bsr done, break pos %u:%u\n", - dev->file, dev->block_num); + Dmsg1(dbglvl, "This bsr done, break pos=%s\n", + dev->print_addr(ed1, sizeof(ed1))); break; } - Dmsg2(900, "After is_bsr_done pos %u:%u\n", dev->file, dev->block_num); + Dmsg2(900, "After is_bsr_done pos %s\n", dev->print_addr(ed1, sizeof(ed1)); } #endif } /* end for loop over records */ - Dmsg2(dbglvl, "After end recs in block. pos=%u:%u\n", dev->file, dev->block_num); + Dmsg1(dbglvl, "After end recs in block. pos=%s\n", dev->print_addr(ed1, sizeof(ed1))); } /* end for loop over blocks */ -// Dmsg2(dbglvl, "Position=(file:block) %u:%u\n", dev->file, dev->block_num); /* Walk down list and free all remaining allocated recs */ while (!recs->empty()) { @@ -318,20 +380,21 @@ static bool try_repositioning(JCR *jcr, DEV_RECORD *rec, DCR *dcr) { BSR *bsr; DEVICE *dev = dcr->dev; + char ed1[50]; bsr = find_next_bsr(jcr->bsr, dev); Dmsg2(dbglvl, "nextbsr=%p mount_next_volume=%d\n", bsr, jcr->bsr->mount_next_volume); if (bsr == NULL && jcr->bsr->mount_next_volume) { Dmsg0(dbglvl, "Would mount next volume here\n"); - Dmsg2(dbglvl, "Current postion (file:block) %u:%u\n", - dev->file, dev->block_num); + Dmsg1(dbglvl, "Current postion Addr=%s\n", + dev->print_addr(ed1, sizeof(ed1))); jcr->bsr->mount_next_volume = false; if (!dev->at_eot()) { /* Set EOT flag to force mount of next Volume */ jcr->mount_next_volume = true; dev->set_eot(); } - rec->Block = 0; + rec->Addr = 0; return true; } if (bsr) { @@ -340,19 +403,18 @@ static bool try_repositioning(JCR *jcr, DEV_RECORD *rec, DCR *dcr) * when find_next_bsr() is fixed not to return a bsr already * completed. */ - uint32_t block, file; - /* TODO: use dev->file_addr ? */ - uint64_t dev_addr = (((uint64_t) dev->file)<<32) | dev->block_num; - uint64_t bsr_addr = get_bsr_start_addr(bsr, &file, &block); + uint64_t dev_addr = dev->get_full_addr(); + uint64_t bsr_addr = get_bsr_start_addr(bsr); + /* Do not position backwards */ if (dev_addr > bsr_addr) { return false; } - Dmsg4(dbglvl, "Try_Reposition from (file:block) %u:%u to %u:%u\n", - dev->file, dev->block_num, file, block); - dev->reposition(dcr, file, block); - rec->Block = 0; - return true; + Dmsg2(dbglvl, "Try_Reposition from addr=%llu to %llu\n", + dev_addr, bsr_addr); + dev->reposition(dcr, bsr_addr); + rec->Addr = 0; + return true; /* We want the next block */ } return false; } @@ -360,25 +422,32 @@ static bool try_repositioning(JCR *jcr, DEV_RECORD *rec, DCR *dcr) /* * Position to the first file on this volume */ -static BSR *position_to_first_file(JCR *jcr, DCR *dcr) +static BSR *position_to_first_file(JCR *jcr, DCR *dcr, BSR *bsr) { - BSR *bsr = NULL; DEVICE *dev = dcr->dev; - uint32_t file, block; + uint64_t bsr_addr; + char ed1[50], ed2[50]; + + Enter(150); /* * Now find and position to first file and block * on this tape. */ - if (jcr->bsr) { - jcr->bsr->reposition = true; /* force repositioning */ - bsr = find_next_bsr(jcr->bsr, dev); + if (bsr) { + bsr->reposition = true; /* force repositioning */ + bsr = find_next_bsr(bsr, dev); - if (get_bsr_start_addr(bsr, &file, &block) > 0) { - Jmsg(jcr, M_INFO, 0, _("Forward spacing Volume \"%s\" to file:block %u:%u.\n"), - dev->VolHdr.VolumeName, file, block); - dev->reposition(dcr, file, block); + if ((bsr_addr=get_bsr_start_addr(bsr)) > 0) { + Jmsg(jcr, M_INFO, 0, _("Forward spacing Volume \"%s\" to addr=%s\n"), + dev->VolHdr.VolumeName, dev->print_addr(ed1, sizeof(ed1), bsr_addr)); + dev->clear_eot(); /* TODO: See where to put this clear() exactly */ + Dmsg2(dbglvl, "pos_to_first_file from addr=%s to %s\n", + dev->print_addr(ed1, sizeof(ed1)), + dev->print_addr(ed2, sizeof(ed2), bsr_addr)); + dev->reposition(dcr, bsr_addr); } } + Leave(150); return bsr; } diff --git a/bacula/src/stored/record.h b/bacula/src/stored/record.h index 36113d6f79..3c5cd18026 100644 --- a/bacula/src/stored/record.h +++ b/bacula/src/stored/record.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -30,23 +30,29 @@ /* Return codes from read_device_volume_label() */ enum { - VOL_NOT_READ = 1, /* Volume label not read */ - VOL_OK, /* volume name OK */ - VOL_NO_LABEL, /* volume not labeled */ - VOL_IO_ERROR, /* volume I/O error */ - VOL_NAME_ERROR, /* Volume name mismatch */ - VOL_CREATE_ERROR, /* Error creating label */ - VOL_VERSION_ERROR, /* Bacula version error */ - VOL_LABEL_ERROR, /* Bad label type */ - VOL_NO_MEDIA, /* Hard error -- no media present */ - VOL_TYPE_ERROR /* Volume type (aligned/non-aligned) error */ + VOL_NOT_READ = 1, /* Volume label not read */ + VOL_OK = 2, /* volume name OK */ + VOL_NO_LABEL = 3, /* volume not labeled */ + VOL_IO_ERROR = 4, /* volume I/O error */ + VOL_NAME_ERROR = 5, /* Volume name mismatch */ + VOL_CREATE_ERROR = 6, /* Error creating label */ + VOL_VERSION_ERROR = 7, /* Bacula version error */ + VOL_LABEL_ERROR = 8, /* Bad label type */ + VOL_NO_MEDIA = 9, /* Hard error -- no media present */ + VOL_TYPE_ERROR = 10 /* Volume type (aligned/non-aligned) error */ }; enum rec_state { st_none, /* No state */ st_header, /* Write header */ st_cont_header, /* Write continuation header */ - st_data /* Write data record */ + st_data, /* Write data record */ + st_adata_blkhdr, /* Adata block header */ + st_adata_rechdr, /* Adata record header */ + st_cont_adata_rechdr, /* Adata continuation rechdr */ + st_adata, /* Write aligned data */ + st_cont_adata, /* Write more aligned data */ + st_adata_label /* Writing adata vol label */ }; @@ -76,9 +82,11 @@ enum rec_state { #define REC_NO_MATCH (1<<3) /* No match on continuation data */ #define REC_CONTINUATION (1<<4) /* Continuation record found */ #define REC_ISTAPE (1<<5) /* Set if device is tape */ +#define REC_ADATA_EMPTY (1<<6) /* Not endough adata in block */ +#define REC_NO_SPLIT (1<<7) /* Do not split this record */ #define is_partial_record(r) ((r)->state_bits & REC_PARTIAL_RECORD) -#define is_block_marked_empty(r) ((r)->state_bits & (REC_BLOCK_EMPTY)) +#define is_block_marked_empty(r) ((r)->state_bits & (REC_BLOCK_EMPTY|REC_ADATA_EMPTY)) /* * DEV_RECORD for reading and writing records. @@ -87,14 +95,16 @@ enum rec_state { * This is the memory structure for the record header. */ struct BSR; /* satisfy forward reference */ +struct VOL_LIST; struct DEV_RECORD { dlink link; /* link for chaining in read_record.c */ /* File and Block are always returned during reading * and writing records. */ uint64_t StreamLen; /* Expected data stream length */ - uint32_t File; /* File number */ - uint32_t Block; /* Block number */ + uint64_t FileOffset; /* Offset of this record inside the file */ + uint64_t StartAddr; /* Start address (when the record is partial) */ + uint64_t Addr; /* Record address */ uint32_t VolSessionId; /* sequential id within this session */ uint32_t VolSessionTime; /* session start time */ int32_t FileIndex; /* sequential file number */ @@ -108,10 +118,14 @@ struct DEV_RECORD { uint32_t remlen; /* temp remainder bytes */ uint32_t data_bytes; /* data_bytes */ uint32_t state_bits; /* state bits */ + uint32_t RecNum; /* Record number in the block */ + uint32_t BlockNumber; /* Block number for this record (used in read_records()) */ + bool invalid; /* The record may be invalid if it was merged with a previous record */ rec_state wstate; /* state of write_record_to_block */ rec_state rstate; /* state of read_record_from_block */ BSR *bsr; /* pointer to bsr that matched */ POOLMEM *data; /* Record data. This MUST be a memory pool item */ + const char *VolumeName; /* From JCR::VolList::VolumeName, freed at the end */ int32_t match_stat; /* bsr match status */ uint32_t last_VolSessionId; /* used in sequencing FI for Vbackup */ uint32_t last_VolSessionTime; @@ -178,6 +192,17 @@ struct Volume_Label { char LabelProg[50]; /* Label program name */ char ProgVersion[50]; /* Program version */ char ProgDate[50]; /* Program build date/time */ + + /* Mostly for aligned volumes, BlockSize also used for dedup volumes */ + char AlignedVolumeName[MAX_NAME_LENGTH+4]; /* Aligned block volume name */ + uint64_t FirstData; /* Offset to first data address */ + uint32_t FileAlignment; /* File alignment factor */ + uint32_t PaddingSize; /* Block padding */ + uint32_t BlockSize; /* Basic block size */ + + /* For Cloud */ + uint64_t ChunkSize; /* Basic chunk size */ + }; #define SER_LENGTH_Volume_Label 1024 /* max serialised length of volume label */ diff --git a/bacula/src/stored/record_read.c b/bacula/src/stored/record_read.c index b69f79c2d0..a08acbe2f7 100644 --- a/bacula/src/stored/record_read.c +++ b/bacula/src/stored/record_read.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -30,9 +30,8 @@ #include "stored.h" /* Imported subroutines */ - -static const int read_dbglvl = 200; -static const int dbgep = 200; /* debug execution path */ +static const int read_dbglvl = 200|DT_VOLUME; +static const int dbgep = 200|DT_VOLUME; /* debug execution path */ /* * Read the header record @@ -48,20 +47,20 @@ static bool read_header(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) char buf1[100], buf2[100]; Dmsg0(dbgep, "=== rpath 1 read_header\n"); + ASSERT2(!block->adata, "Block is adata. Wrong!"); /* Clear state flags */ rec->state_bits = 0; if (block->dev->is_tape()) { rec->state_bits |= REC_ISTAPE; } - rec->Block = ((DEVICE *)block->dev)->EndBlock; - rec->File = ((DEVICE *)block->dev)->EndFile; + rec->Addr = ((DEVICE *)block->dev)->EndAddr; /* * Get the header. There is always a full header, * otherwise we find it in the next block. */ - Dmsg3(read_dbglvl, "Block=%d Ver=%d block_len=%u\n", - block->BlockNumber, block->BlockVer, block->block_len); + Dmsg4(read_dbglvl, "adata=%d Block=%d Ver=%d block_len=%u\n", + block->adata, block->BlockNumber, block->BlockVer, block->block_len); if (block->BlockVer == 1) { rhl = RECHDR1_LENGTH; } else { @@ -84,6 +83,10 @@ static bool read_header(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) unser_int32(Stream); unser_uint32(rec->data_bytes); + if (dcr->dev->have_adata_header(dcr, rec, FileIndex, Stream, VolSessionId)) { + return true; + } + block->bufp += rhl; block->binbuf -= rhl; rec->remlen -= rhl; @@ -185,6 +188,7 @@ static void read_data(DEV_BLOCK *block, DEV_RECORD *rec) char buf1[100], buf2[100]; Dmsg0(dbgep, "=== rpath 22 read_data\n"); + ASSERT2(!block->adata, "Block is adata. Wrong!"); /* * At this point, we have read the header, now we * must transfer as much of the data record as @@ -201,8 +205,8 @@ static void read_data(DEV_BLOCK *block, DEV_RECORD *rec) block->binbuf -= rec->data_bytes; rec->data_len += rec->data_bytes; rec->remainder = 0; - Dmsg5(190, "Rdata full FI=%s SessId=%d Strm=%s len=%d block=%p\n", - FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, + Dmsg6(190, "Rdata full adata=%d FI=%s SessId=%d Strm=%s len=%d block=%p\n", + block->adata, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len, block); } else { @@ -229,13 +233,25 @@ static void read_data(DEV_BLOCK *block, DEV_RECORD *rec) */ bool read_record_from_block(DCR *dcr, DEV_RECORD *rec) { + bool save_adata = dcr->dev->adata; bool rtn; Dmsg0(dbgep, "=== rpath 1 Enter read_record_from block\n"); + + /* Update the Record number only if we have a new record */ + if (rec->remainder == 0) { + rec->RecNum = dcr->block->RecNum; + rec->VolumeName = dcr->CurrentVol->VolumeName; /* From JCR::VolList, freed at the end */ + rec->Addr = rec->StartAddr = dcr->block->BlockAddr; + } + + /* We read the next record */ + dcr->block->RecNum++; + for ( ;; ) { switch (rec->rstate) { case st_none: - dump_block(dcr->ameta_block, "st_none"); + dump_block(dcr->dev, dcr->ameta_block, "st_none"); case st_header: Dmsg0(dbgep, "=== rpath 33 st_header\n"); dcr->set_ameta(); @@ -254,6 +270,31 @@ bool read_record_from_block(DCR *dcr, DEV_RECORD *rec) rec->rstate = st_header; /* next pass look for a header */ goto get_out; + case st_adata_blkhdr: + dcr->set_adata(); + dcr->dev->read_adata_block_header(dcr); + rec->rstate = st_header; + continue; + + case st_adata_rechdr: + Dmsg0(dbgep, "=== rpath 35 st_adata_rechdr\n"); + if (!dcr->dev->read_adata_record_header(dcr, dcr->block, rec)) { /* sets state */ + Dmsg0(dbgep, "=== rpath 36 failed read_adata rechdr\n"); + Dmsg0(100, "read_link returned EOF.\n"); + goto fail_out; + } + continue; + + case st_adata: + switch (dcr->dev->read_adata(dcr, rec)) { + case -1: + goto fail_out; + case 0: + continue; + case 1: + goto get_out; + } + default: Dmsg0(dbgep, "=== rpath 50 default\n"); Dmsg0(0, "======= In default !!!!!\n"); @@ -263,15 +304,20 @@ bool read_record_from_block(DCR *dcr, DEV_RECORD *rec) } get_out: char buf1[100], buf2[100]; - Dmsg5(read_dbglvl, "read_rec return: FI=%s Strm=%s len=%d rem=%d remainder=%d\n", + Dmsg6(read_dbglvl, "read_rec return: FI=%s Strm=%s len=%d rem=%d remainder=%d Num=%d\n", FI_to_ascii(buf1, rec->FileIndex), stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len, - rec->remlen, rec->remainder); + rec->remlen, rec->remainder, rec->RecNum); rtn = true; goto out; fail_out: rec->rstate = st_none; rtn = false; out: + if (save_adata) { + dcr->set_adata(); + } else { + dcr->set_ameta(); + } return rtn; } diff --git a/bacula/src/stored/record_util.c b/bacula/src/stored/record_util.c index 7c3c54dad5..a326c14ad2 100644 --- a/bacula/src/stored/record_util.c +++ b/bacula/src/stored/record_util.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -149,6 +149,10 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "contENCRYPTED-MACOS-RSRC"; case STREAM_PLUGIN_NAME: return "contPLUGIN-NAME"; + case STREAM_ADATA_BLOCK_HEADER: + return "contADATA-BLOCK-HEADER"; + case STREAM_ADATA_RECORD_HEADER: + return "contADATA-RECORD-HEADER"; default: sprintf(buf, "%d", -stream); @@ -217,13 +221,24 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "ENCRYPTED-WIN32-COMPRESSED"; case STREAM_ENCRYPTED_MACOS_FORK_DATA: return "ENCRYPTED-MACOS-RSRC"; - + case STREAM_ADATA_BLOCK_HEADER: + return "ADATA-BLOCK-HEADER"; + case STREAM_ADATA_RECORD_HEADER: + return "ADATA-RECORD-HEADER"; default: sprintf(buf, "%d", stream); return buf; } } +const char *stream_to_ascii_ex(char *buf, int stream, int fi) +{ + if (fi < 0) { + return stream_to_ascii(buf, stream, fi); + } + const char *p = stream_to_ascii(buf, stream, fi); + return p; +} /* * Return a new record entity */ @@ -241,13 +256,16 @@ DEV_RECORD *new_record(void) void empty_record(DEV_RECORD *rec) { - rec->File = rec->Block = 0; + rec->RecNum = 0; + rec->StartAddr = rec->Addr = 0; rec->VolSessionId = rec->VolSessionTime = 0; rec->FileIndex = rec->Stream = 0; rec->data_len = rec->remainder = 0; - rec->state_bits &= ~(REC_PARTIAL_RECORD|REC_BLOCK_EMPTY|REC_NO_MATCH|REC_CONTINUATION); + rec->state_bits &= ~(REC_PARTIAL_RECORD|REC_ADATA_EMPTY|REC_BLOCK_EMPTY|REC_NO_MATCH|REC_CONTINUATION); + rec->FileOffset = 0; rec->wstate = st_none; rec->rstate = st_none; + rec->VolumeName = NULL; } /* @@ -265,6 +283,20 @@ void free_record(DEV_RECORD *rec) Dmsg0(950, "Leave free_record.\n"); } +void dump_record(DEV_RECORD *rec) +{ + char buf[32]; + Dmsg11(100|DT_VOLUME, "Dump record %s 0x%p:\n\tStart=%lld addr=%lld #%d\n" + "\tVolSess: %ld:%ld\n" + "\tFileIndex: %ld\n" + "\tStream: 0x%lx\n\tLen: %ld\n\tData: %s\n", + rec, NPRT(rec->VolumeName), + rec->StartAddr, rec->Addr, rec->RecNum, + rec->VolSessionId, rec->VolSessionTime, rec->FileIndex, + rec->Stream, rec->data_len, + asciidump(rec->data, rec->data_len, buf, sizeof(buf))); +} + /* * Test if we can write whole record to the block * @@ -294,5 +326,10 @@ bool can_write_record_to_block(DEV_BLOCK *block, DEV_RECORD *rec) uint64_t get_record_address(DEV_RECORD *rec) { - return ((uint64_t)rec->File)<<32 | rec->Block; + return rec->Addr; +} + +uint64_t get_record_start_address(DEV_RECORD *rec) +{ + return rec->StartAddr; } diff --git a/bacula/src/stored/record_write.c b/bacula/src/stored/record_write.c index bebe0740c6..4ac4879ba2 100644 --- a/bacula/src/stored/record_write.c +++ b/bacula/src/stored/record_write.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -22,6 +22,7 @@ * * Kern Sibbald, April MMI * added BB02 format October MMII + * added aligned format November MMXII * */ @@ -43,44 +44,20 @@ struct rechdr { }; /* - * Flush block to disk - */ -bool flush_block(DCR *dcr) -{ - bool rtn = false; - - if (!is_block_empty(dcr->block)) { - Dmsg0(dbgep, "=== wpath 53 flush_block\n"); - Dmsg3(190, "Call flush_block BlockAddr=%lld nbytes=%d block=%x\n", - dcr->block->BlockAddr, dcr->block->binbuf, dcr->block); - dump_block(dcr->block, "Flush_block"); - if (dcr->jcr->is_canceled() || !dcr->write_block_to_device()) { - Dmsg0(dbgep, "=== wpath 54 flush_block\n"); - Dmsg0(190, "Failed to write block to device, return false.\n"); - goto get_out; - } - empty_block(dcr->block); - } - rtn = true; - -get_out: - return rtn; -} - -/* - * Write a header record to the block. + * Write an ameta (normal) header record to the block. */ static bool write_header_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) { ser_declare; Dmsg0(dbgep, "=== wpath 11 write_header_to_block\n"); + ASSERT2(!block->adata, "Attempt to write header to adata block!"); rec->remlen = block->buf_len - block->binbuf; /* Require enough room to write a full header */ if (rec->remlen < WRITE_RECHDR_LENGTH) { Dmsg0(dbgep, "=== wpath 12 write_header_to_block\n"); - Dmsg4(190, "Fail remlen=%d<%d reclen buf_len=%d binbuf=%d\n", - rec->remlen, WRITE_RECHDR_LENGTH, block->buf_len, block->binbuf); + Dmsg5(190, "remlenadata, rec->remlen, WRITE_RECHDR_LENGTH, block->buf_len, block->binbuf); rec->remainder = rec->data_len + WRITE_RECHDR_LENGTH; return false; } @@ -100,6 +77,8 @@ static bool write_header_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) block->bufp += WRITE_RECHDR_LENGTH; block->binbuf += WRITE_RECHDR_LENGTH; + + block->RecNum++; rec->remlen -= WRITE_RECHDR_LENGTH; rec->remainder = rec->data_len; if (rec->FileIndex > 0) { @@ -112,12 +91,12 @@ static bool write_header_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) block->LastIndex = rec->FileIndex; } - //dump_block(block, "Add header"); + //dump_block(dcr->dev, block, "Add header"); return true; } /* - * If the prior block was not big enough to hold the + * If the prior ameta block was not big enough to hold the * whole record, write a continuation header record. */ static void write_continue_header_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) @@ -125,7 +104,14 @@ static void write_continue_header_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECOR ser_declare; Dmsg0(dbgep, "=== wpath 17 write_cont_hdr_to_block\n"); + ASSERT2(!block->adata, "Attempt to write adata header!"); rec->remlen = block->buf_len - block->binbuf; + + /* No space left to write the continue header */ + if (rec->remlen == 0) { + return; + } + /* * We have unwritten bytes from a previous * time. Presumably we have a new buffer (possibly @@ -177,14 +163,21 @@ static void write_continue_header_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECOR } block->LastIndex = rec->FileIndex; } - //dump_block(block, "Add cont header"); + if (block->adata) { + Dmsg3(150, "=== write_cont_hdr ptr=%p begin=%p off=%d\n", block->bufp, + block->buf, block->bufp-block->buf); + } + block->RecNum++; + //dump_block(dcr->dev, block, "Add cont header"); } /* + * Now write non-aligned data to an ameta block */ static bool write_data_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) { Dmsg0(dbgep, "=== wpath 24 write_data_to_block\n"); + ASSERT2(!block->adata, "Attempt to write adata to metadata file!"); rec->remlen = block->buf_len - block->binbuf; /* Write as much of data as possible */ if (rec->remlen >= rec->remainder) { @@ -195,6 +188,9 @@ static bool write_data_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) block->binbuf += rec->remainder; rec->remainder = 0; } else { + if (rec->state_bits & REC_NO_SPLIT) { + return false; /* do not split record */ + } Dmsg0(dbgep, "=== wpath 26 write_data_to_block\n"); memcpy(block->bufp, rec->data+rec->data_len-rec->remainder, rec->remlen); @@ -203,6 +199,12 @@ static bool write_data_to_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) rec->remainder -= rec->remlen; return false; /* did partial transfer */ } + if (block->adata) { + /* Adata label data */ + Dmsg3(190, "write_data adata=%d blkAddr=%lld off=%d\n", + block->adata, block->BlockAddr, block->bufp-block->buf); + } + //dump_block(dcr->dev, block, "Add data/adata"); return true; } @@ -252,23 +254,29 @@ bool DCR::write_record(DEV_RECORD *rec) * non-zero), and 2. The remaining bytes to write may not * all fit into the block. * + * Ensure we leave with same ameta/adata set as enter. */ bool write_record_to_block(DCR *dcr, DEV_RECORD *rec) { char buf1[100], buf2[100]; + bool save_adata = dcr->block->adata; bool rtn; Enter(dbgel); Dmsg0(dbgep, "=== wpath 35 enter write_record_to_block\n"); - Dmsg7(200, "write_record_to_block() state=%d FI=%s SessId=%d" + Dmsg7(250, "write_record_to_block() state=%d FI=%s SessId=%d" " Strm=%s len=%d rem=%d remainder=%d\n", rec->wstate, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len, rec->remlen, rec->remainder); - Dmsg4(160, "write_rec Strm=%s len=%d rem=%d remainder=%d\n", + Dmsg4(250, "write_rec Strm=%s len=%d rem=%d remainder=%d\n", stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len, rec->remlen, rec->remainder); + if (!dcr->dev->do_pre_write_checks(dcr, rec)) { + goto fail_out; + } + for ( ;; ) { Dmsg0(dbgep, "=== wpath 37 top of for loop\n"); ASSERT(dcr->block->binbuf == (uint32_t) (dcr->block->bufp - dcr->block->buf)); @@ -279,11 +287,21 @@ bool write_record_to_block(DCR *dcr, DEV_RECORD *rec) Dmsg0(dbgep, "=== wpath 38 st_none\n"); /* Figure out what to do */ rec->wstate = st_header; + /* If labeling adata, special path */ + if (dcr->adata_label) { + Dmsg1(dbgep, "=== wpath adata_label set adata=%d\n", dcr->dev->adata); + rec->wstate = st_adata_label; + continue; + } if (rec->FileIndex < 0) { - /* Label record */ + /* Label record -- ameta label */ + Dmsg3(dbgep, "=== wpath label adata=%d Strm=%d FI=%d\n", + dcr->dev->adata, rec->Stream, rec->FileIndex); rec->wstate = st_header; continue; } + + dcr->dev->select_data_stream(dcr, rec); continue; /* go to next state */ case st_header: @@ -295,7 +313,8 @@ bool write_record_to_block(DCR *dcr, DEV_RECORD *rec) * that did not previously fit into the block. */ Dmsg0(dbgep, "=== wpath 42 st_header\n"); - if (!write_header_to_block(dcr, dcr->block, rec)) { + dcr->set_ameta(); + if (!write_header_to_block(dcr, dcr->ameta_block, rec)) { Dmsg0(dbgep, "=== wpath 43 st_header\n"); rec->wstate = st_cont_header; goto fail_out; @@ -306,7 +325,8 @@ bool write_record_to_block(DCR *dcr, DEV_RECORD *rec) case st_cont_header: Dmsg0(dbgep, "=== wpath 45 st_cont_header\n"); - write_continue_header_to_block(dcr, dcr->block, rec); + dcr->set_ameta(); + write_continue_header_to_block(dcr, dcr->ameta_block, rec); rec->wstate = st_data; if (rec->remlen == 0) { Dmsg0(dbgep, "=== wpath 46 st_cont_header\n"); @@ -325,18 +345,61 @@ bool write_record_to_block(DCR *dcr, DEV_RECORD *rec) * may not have enough room to transfer the whole this time. */ Dmsg0(dbgep, "=== wpath 47 st_data\n"); + dcr->set_ameta(); if (rec->remainder > 0) { Dmsg0(dbgep, "=== wpath 48 st_data\n"); - if (!write_data_to_block(dcr, dcr->block, rec)) { + if (!write_data_to_block(dcr, dcr->ameta_block, rec)) { Dmsg0(dbgep, "=== wpath 49 st_data\n"); - rec->wstate = st_cont_header; + if (rec->state_bits & REC_NO_SPLIT) { + rec->wstate = st_header; + } else { + rec->wstate = st_cont_header; + } goto fail_out; } } + rec->state_bits &= ~REC_NO_SPLIT; /* clear possible no split bit */ rec->remainder = 0; /* did whole transfer */ rec->wstate = st_none; goto get_out; + case st_adata_label: + if (!dcr->dev->write_adata_label(dcr, rec)) { + goto fail_out; + } + goto get_out; + + /* + * We come here only once for each record + */ + case st_adata: + dcr->dev->write_adata(dcr, rec); + continue; + + case st_cont_adata: + dcr->dev->write_cont_adata(dcr, rec); + continue; + + /* + * Note, the following two cases are handled differently + * in write_adata_record_header() so take care if you want to + * eliminate one of them. + */ + case st_cont_adata_rechdr: + Dmsg2(200, "=== cont rechdr remainder=%d reclen=%d\n", rec->remainder, dcr->adata_block->reclen); + Dmsg0(200, "st_cont_adata_rechdr\n"); + /* Fall through wanted */ + case st_adata_rechdr: + switch(dcr->dev->write_adata_rechdr(dcr, rec)) { + case -1: + goto fail_out; + case 0: + continue; + case 1: + goto get_out; + } + break; + default: Dmsg0(dbgep, "=== wpath 67!!!! default\n"); Dmsg0(50, "Something went wrong. Default state.\n"); @@ -350,6 +413,11 @@ get_out: fail_out: rtn = false; out: + if (save_adata) { + dcr->set_adata(); + } else { + dcr->set_ameta(); + } Leave(dbgel); return rtn; } diff --git a/bacula/src/stored/reserve.c b/bacula/src/stored/reserve.c index aca0cc4f94..eb04e620f4 100644 --- a/bacula/src/stored/reserve.c +++ b/bacula/src/stored/reserve.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -42,6 +42,7 @@ static bool use_device_cmd(JCR *jcr); static int reserve_device(RCTX &rctx); static void pop_reserve_messages(JCR *jcr); static void queue_reserve_message(JCR *jcr); +static int is_pool_ok(DCR *dcr); //void switch_device(DCR *dcr, DEVICE *dev); /* Requests from the Director daemon */ @@ -52,7 +53,7 @@ static char use_device[] = "use device=%127s\n"; /* Responses sent to Director daemon */ static char OK_device[] = "3000 OK use device device=%s\n"; static char NO_device[] = "3924 Device \"%s\" not in SD Device" - " resources or no matching Media Type.\n"; + " resources or no matching Media Type or is disabled.\n"; static char BAD_use[] = "3913 Bad use command: %s\n"; /* @@ -106,24 +107,36 @@ void DCR::clear_reserved() if (m_reserved) { m_reserved = false; dev->dec_reserved(); - Dmsg2(dbglvl, "Dec reserve=%d dev=%s\n", dev->num_reserved(), dev->print_name()); + Dmsg3(dbglvl, "Dec reserve=%d writers=%d dev=%s\n", dev->num_reserved(), + dev->num_writers, dev->print_name()); + if (dev->num_reserved() == 0) { + dev->reserved_pool_name[0] = 0; + } } } void DCR::set_reserved_for_append() { + if (dev->num_reserved() == 0) { + bstrncpy(dev->reserved_pool_name, pool_name, sizeof(dev->reserved_pool_name)); + Dmsg1(dbglvl, "Set reserve pool: %s\n", pool_name); + } m_reserved = true; dev->set_append_reserve(); - Dmsg2(dbglvl, "Inc reserve=%d dev=%s\n", dev->num_reserved(), dev->print_name()); dev->inc_reserved(); + Dmsg3(dbglvl, "Inc reserve=%d writers=%d dev=%s\n", dev->num_reserved(), + dev->num_writers, dev->print_name()); } void DCR::set_reserved_for_read() { - m_reserved = true; - dev->set_read_reserve(); - Dmsg2(dbglvl, "Inc reserve=%d dev=%s\n", dev->num_reserved(), dev->print_name()); - dev->inc_reserved(); + /* Called for each volume read, no need to increment each time */ + if (!m_reserved) { + m_reserved = true; + dev->set_read_reserve(); + dev->inc_reserved(); + Dmsg2(dbglvl, "Inc reserve=%d dev=%s\n", dev->num_reserved(), dev->print_name()); + } } /* @@ -170,6 +183,75 @@ bool use_cmd(JCR *jcr) return true; } +/* + * Storage search options + */ +struct store_opts_t { + bool PreferMountedVols; + bool exact_match; + bool autochanger_only; + bool try_low_use_drive; + bool any_drive; +}; + +/* *** Old pre-8.8 algorithm *** not used here + * jcr->PreferMountedVols = true + * MntVol exact chgonly lowuse any + * 1 * {true, true, false, false, false}, + * 2 * {true, false, true, false, false}, + * 3 * {true, false, true, false, true}, + + * jcr->PreferMountedVols = false + * MntVol exact chgonly lowuse any + * 4 * {false, false, true, false, false}, + * 5 * {false, false, true, true, false}, * low instantaneous use * + * 6 * {false, false, false, true, false}, + * 7 * {true, true, false, false, false}, + * 8 * {true, false, true, false, false}, + * 9 * {true, false, true, false, true} + *** End old pre-8.8 algorithm *** + */ + +store_opts_t store_opts[] = { +/* jcr->PreferMountedVols = true */ +/* MntVol exact chgonly lowuse any */ +/* 1 */ {true, true, false, true, false}, /* low global use */ +/* 2 */ {true, false, true, true, false}, +/* 3 */ {true, false, true, false, true}, +/* 4 */ {true, true, false, true, false}, +/* 5 */ {false, true, false, false, false}, +/* 6 */ {false, false, false, false, false}, +/* 7 */ {true, false, false, false, true}, + +/* jcr->PreferMountedVols = false */ +/* MntVol exact chgonly lowuse any */ +/* 8 */ {false, false, true, true, false}, +/* 9 */ {false, false, true, false, false}, +/* 10 */ {false, false, false, true, false}, +/* 11 */ {true, true, false, true, true}, +/* 12 */ {true, false, true, true, false}, +/* 13 */ {true, false, true, false, true} +}; + +static void set_options(RCTX &rctx, int num) +{ + rctx.PreferMountedVols = store_opts[num-1].PreferMountedVols; + rctx.exact_match = store_opts[num-1].exact_match; + rctx.autochanger_only = store_opts[num-1].autochanger_only; + rctx.try_low_use_drive = store_opts[num-1].try_low_use_drive; + rctx.any_drive = store_opts[num-1].any_drive; + rctx.low_use_drive = NULL; +} + +static void prt_options(RCTX &rctx, int num) +{ + Dmsg6(dbglvl, "Inx=%d mntVol=%d exact=%d chgonly=%d low_use=%d any=%d\n", + num, rctx.PreferMountedVols, rctx.exact_match, rctx.autochanger_only, + rctx.try_low_use_drive, rctx.any_drive); +} + + + /* * We get the following type of information: * @@ -270,7 +352,7 @@ static bool use_device_cmd(JCR *jcr) * Then for each of the Storage resources, we have a list of * device names that were given. * - * Wiffle through them and find one that can do the backup. + * Wiffle through Storage resources sent to us and find one that can do the backup. */ if (ok) { int wait_for_device_retries = 0; @@ -286,57 +368,30 @@ static bool use_device_cmd(JCR *jcr) } lock_reservations(); for ( ; !fail && !job_canceled(jcr); ) { + int i; pop_reserve_messages(jcr); rctx.suitable_device = false; rctx.have_volume = false; rctx.VolumeName[0] = 0; rctx.any_drive = false; - if (!jcr->PreferMountedVols) { - /* - * Here we try to find a drive that is not used. - * This will maximize the use of available drives. - * - */ - rctx.num_writers = 20000000; /* start with impossible number */ - rctx.low_use_drive = NULL; - rctx.PreferMountedVols = false; - rctx.exact_match = false; - rctx.autochanger_only = true; - if ((ok = find_suitable_device_for_job(jcr, rctx))) { - break; - } - /* Look through all drives possibly for low_use drive */ - if (rctx.low_use_drive) { - rctx.try_low_use_drive = true; + if (jcr->PreferMountedVols) { + for (i=1; i<=7; i++) { + set_options(rctx, i); + prt_options(rctx, i); if ((ok = find_suitable_device_for_job(jcr, rctx))) { break; } - rctx.try_low_use_drive = false; } - rctx.autochanger_only = false; - if ((ok = find_suitable_device_for_job(jcr, rctx))) { - break; + } else { + for (i=8; i<=13; i++) { + set_options(rctx, i); + prt_options(rctx, i); + if ((ok = find_suitable_device_for_job(jcr, rctx))) { + break; + } } } - /* - * Now we look for a drive that may or may not be in - * use. - */ - /* Look for an exact Volume match all drives */ - rctx.PreferMountedVols = true; - rctx.exact_match = true; - rctx.autochanger_only = false; - if ((ok = find_suitable_device_for_job(jcr, rctx))) { - break; - } - /* Look for any mounted drive */ - rctx.exact_match = false; - if ((ok = find_suitable_device_for_job(jcr, rctx))) { - break; - } - /* Try any drive */ - rctx.any_drive = true; - if ((ok = find_suitable_device_for_job(jcr, rctx))) { + if (ok) { break; } /* Keep reservations locked *except* during wait_for_device() */ @@ -350,9 +405,9 @@ static bool use_device_cmd(JCR *jcr) */ if (repeat++ > 1) { /* try algorithm 3 times */ bmicrosleep(30, 0); /* wait a bit */ - Dmsg1(100, "repeat reserve algorithm JobId=%d\n", jcr->JobId); + Dmsg1(dbglvl, "repeat reserve algorithm JobId=%d\n", jcr->JobId); } else if (!rctx.suitable_device || !wait_for_any_device(jcr, wait_for_device_retries)) { - Dmsg0(100, "Fail. !suitable_device || !wait_for_device\n"); + Dmsg0(dbglvl, "Fail. !suitable_device || !wait_for_device\n"); fail = true; } lock_reservations(); @@ -428,12 +483,11 @@ bool find_suitable_device_for_job(JCR *jcr, RCTX &rctx) Dmsg1(dbglvl, "vol=%s no dev\n", vol->vol_name); continue; } - /* Check with Director if this Volume is OK */ bstrncpy(dcr->VolumeName, vol->vol_name, sizeof(dcr->VolumeName)); - if (!dir_get_volume_info(dcr, GET_VOL_INFO_FOR_WRITE)) { + /* Check with Director if this Volume is OK */ + if (!dir_get_volume_info(dcr, dcr->VolumeName, GET_VOL_INFO_FOR_WRITE)) { continue; } - Dmsg1(dbglvl, "vol=%s OK for this job\n", vol->vol_name); foreach_alist(store, dirstore) { int stat; @@ -442,7 +496,6 @@ bool find_suitable_device_for_job(JCR *jcr, RCTX &rctx) /* Found a device, try to use it */ rctx.device_name = device_name; rctx.device = vol->dev->device; - if (vol->dev->read_only) { continue; } @@ -457,29 +510,25 @@ bool find_suitable_device_for_job(JCR *jcr, RCTX &rctx) vol->dev->device->hdr.name, device_name); continue; } - bstrncpy(rctx.VolumeName, vol->vol_name, sizeof(rctx.VolumeName)); rctx.have_volume = true; /* Try reserving this device and volume */ - Dmsg2(dbglvl, "try vol=%s on device=%s\n", rctx.VolumeName, device_name); + Dmsg2(dbglvl, "Try reserve vol=%s on device=%s\n", rctx.VolumeName, device_name); stat = reserve_device(rctx); if (stat == 1) { /* found available device */ - Dmsg1(dbglvl, "Suitable device found=%s\n", device_name); + Dmsg1(dbglvl, "Device reserved=%s\n", device_name); ok = true; - break; - } else if (stat == 0) { /* device busy */ - Dmsg1(dbglvl, "Suitable device=%s, busy: not use\n", device_name); } else { - /* otherwise error */ + /* Error or no suitable device found */ Dmsg0(dbglvl, "No suitable device found.\n"); + rctx.have_volume = false; + rctx.VolumeName[0] = 0; } - rctx.have_volume = false; - rctx.VolumeName[0] = 0; } if (ok) { break; } - } + } /* end of loop over storages */ if (ok) { break; } @@ -487,7 +536,7 @@ bool find_suitable_device_for_job(JCR *jcr, RCTX &rctx) Dmsg0(dbglvl, "lock volumes\n"); free_temp_vol_list(temp_vol_list); - } + } /* End test for vol_list, ... */ if (ok) { Dmsg1(dbglvl, "OK dev found. Vol=%s from in-use vols list\n", rctx.VolumeName); return true; @@ -545,26 +594,67 @@ int search_res_for_device(RCTX &rctx) if (strcmp(rctx.device_name, changer->hdr.name) == 0) { /* Try each device in this AutoChanger */ foreach_alist(rctx.device, changer->device) { - Dmsg1(dbglvl, "Try changer device %s\n", rctx.device->hdr.name); + Dmsg1(dbglvl, "Top try changer device %s\n", rctx.device->hdr.name); if (rctx.store->append && rctx.device->read_only) { continue; } if (!rctx.device->autoselect) { - Dmsg1(100, "Device %s not autoselect skipped.\n", + Dmsg1(dbglvl, "Device %s not autoselect skipped.\n", rctx.device->hdr.name); continue; /* device is not available */ } - stat = reserve_device(rctx); - if (stat != 1) { /* try another device */ - continue; - } - /* Debug code */ - if (rctx.store->append) { - Dmsg2(dbglvl, "Device %s reserved=%d for append.\n", + if (rctx.try_low_use_drive) { + if (!rctx.low_use_drive) { + rctx.low_use_drive = rctx.device->dev; + Dmsg2(dbglvl, "Set low_use usage=%lld drv=%s\n", + rctx.low_use_drive->usage, + rctx.low_use_drive->print_name()); + } else if ((rctx.low_use_drive->usage > rctx.device->dev->usage) || + (rctx.low_use_drive->usage == rctx.device->dev->usage && + rctx.low_use_drive->num_reserved() > rctx.device->dev->num_reserved())) { + rctx.low_use_drive = rctx.device->dev; + Dmsg2(dbglvl, "Reset low_use usage=%lld drv=%s\n", + rctx.low_use_drive->usage, + rctx.low_use_drive->print_name()); + } else { + Dmsg2(dbglvl, "Skip low_use usage=%lld drv=%s\n", + rctx.low_use_drive->usage, + rctx.low_use_drive->print_name()); + } + } else { + Dmsg2(dbglvl, "try reserve vol=%s on device=%s\n", rctx.VolumeName, rctx.device->hdr.name); + stat = reserve_device(rctx); + if (stat != 1) { /* try another device */ + continue; + } + /* Debug code */ + if (rctx.store->append) { + Dmsg2(dbglvl, "Device %s reserved=%d for append.\n", rctx.device->hdr.name, rctx.jcr->dcr->dev->num_reserved()); + } else { + Dmsg2(dbglvl, "Device %s reserved=%d for read.\n", + rctx.device->hdr.name, rctx.jcr->read_dcr->dev->num_reserved()); + } + return stat; + } + } + /* If found a drive try to reserve it */ + if (rctx.try_low_use_drive && rctx.low_use_drive) { + rctx.device = rctx.low_use_drive->device; + Dmsg2(dbglvl, "Try reserve vol=%s on device=%s\n", rctx.VolumeName, rctx.device->hdr.name); + stat = reserve_device(rctx); + if (stat == 1) { + /* Debug code */ + if (rctx.store->append) { + Dmsg3(dbglvl, "JobId=%d device %s reserved=%d for append.\n", + rctx.jcr->JobId, rctx.device->hdr.name, rctx.jcr->dcr->dev->num_reserved()); + } else { + Dmsg3(dbglvl, "JobId=%d device %s reserved=%d for read.\n", + rctx.jcr->JobId, rctx.device->hdr.name, rctx.jcr->read_dcr->dev->num_reserved()); + } } else { - Dmsg2(dbglvl, "Device %s reserved=%d for read.\n", - rctx.device->hdr.name, rctx.jcr->read_dcr->dev->num_reserved()); + Dmsg2(dbglvl, "Reserve for %s failed for JobId=%d.\n", + rctx.store->append ? "append" : "read", rctx.jcr->JobId); } return stat; } @@ -577,6 +667,7 @@ int search_res_for_device(RCTX &rctx) Dmsg1(dbglvl, "Try match res=%s\n", rctx.device->hdr.name); /* Find resource, and make sure we were able to open it */ if (strcmp(rctx.device_name, rctx.device->hdr.name) == 0) { + Dmsg2(dbglvl, "Try reserve vol=%s on device=%s\n", rctx.VolumeName, rctx.device->hdr.name); stat = reserve_device(rctx); if (stat != 1) { /* try another device */ continue; @@ -607,12 +698,8 @@ static bool is_vol_in_autochanger(RCTX &rctx, VOLRES *vol) { AUTOCHANGER *changer = vol->dev->device->changer_res; - if (!changer) { - return false; - } - /* Find resource, and make sure we were able to open it */ - if (strcmp(rctx.device_name, changer->hdr.name) == 0) { + if (changer && strcmp(rctx.device_name, changer->hdr.name) == 0) { Dmsg1(dbglvl, "Found changer device %s\n", vol->dev->device->hdr.name); return true; } @@ -686,13 +773,13 @@ static int reserve_device(RCTX &rctx) if (!ok) { goto bail_out; } - rctx.jcr->dcr = dcr; Dmsg5(dbglvl, "Reserved=%d dev_name=%s mediatype=%s pool=%s ok=%d\n", dcr->dev->num_reserved(), dcr->dev_name, dcr->media_type, dcr->pool_name, ok); - Dmsg3(dbglvl, "Vol=%s num_writers=%d, have_vol=%d\n", - rctx.VolumeName, dcr->dev->num_writers, rctx.have_volume); + Dmsg4(dbglvl, "Vol=%s num_writers=%d, reserved=%d have_vol=%d\n", + rctx.VolumeName, dcr->dev->num_writers, dcr->dev->num_reserved(), + rctx.have_volume); if (rctx.have_volume) { Dmsg0(dbglvl, "Call reserve_volume for append.\n"); if (reserve_volume(dcr, rctx.VolumeName)) { @@ -708,6 +795,12 @@ static int reserve_device(RCTX &rctx) bstrncpy(rctx.VolumeName, dcr->VolumeName, sizeof(rctx.VolumeName)); rctx.have_volume = true; Dmsg1(dbglvl, "looking for Volume=%s\n", rctx.VolumeName); + if (!dcr->can_i_use_volume() || !is_pool_ok(dcr)) { + rctx.have_volume = false; + rctx.VolumeName[0] = 0; + dcr->unreserve_device(false); + goto bail_out; + } } else { dcr->dev->clear_wait(); Dmsg0(dbglvl, "No next volume found\n"); @@ -794,7 +887,7 @@ bail_out: * Note, in reserving a device, if the device is for the * same pool and the same pool type, then it is acceptable. * The Media Type has already been checked. If we are - * the first tor reserve the device, we put the pool + * the first to reserve the device, we put the pool * name and pool type in the device record. */ static bool reserve_device_for_append(DCR *dcr, RCTX &rctx) @@ -907,14 +1000,15 @@ static bool is_max_jobs_ok(DCR *dcr) DEVICE *dev = dcr->dev; JCR *jcr = dcr->jcr; - Dmsg5(dbglvl, "MaxJobs=%d Jobs=%d reserves=%d Status=%s Vol=%s\n", + Dmsg6(dbglvl, "MaxJobs=%d VolCatJobs=%d writers=%d reserves=%d Status=%s Vol=%s\n", dcr->VolCatInfo.VolCatMaxJobs, - dcr->VolCatInfo.VolCatJobs, dev->num_reserved(), + dcr->VolCatInfo.VolCatJobs, + dev->num_writers, dev->num_reserved(), dcr->VolCatInfo.VolCatStatus, dcr->VolumeName); /* Limit max concurrent jobs on this drive */ - if (dev->max_concurrent_jobs > 0 && dev->max_concurrent_jobs <= - (uint32_t)(dev->num_writers + dev->num_reserved())) { + if (dev->max_concurrent_jobs > 0 && (int)dev->max_concurrent_jobs <= + (dev->num_writers + dev->num_reserved())) { /* Max Concurrent Jobs depassed or already reserved */ Mmsg(jcr->errmsg, _("3609 JobId=%u Max concurrent jobs=%d exceeded on %s device %s.\n"), (uint32_t)jcr->JobId, dev->max_concurrent_jobs, @@ -926,9 +1020,14 @@ static bool is_max_jobs_ok(DCR *dcr) if (strcmp(dcr->VolCatInfo.VolCatStatus, "Recycle") == 0) { return true; } - - if (dcr->VolCatInfo.VolCatMaxJobs > 0 && dcr->VolCatInfo.VolCatMaxJobs <= - (dcr->VolCatInfo.VolCatJobs + dev->num_reserved())) { + if (!dev->allow_maxbytes_concurrency(dcr)) { + queue_reserve_message(jcr); + Dmsg1(dbglvl, "reserve dev failed: %s", jcr->errmsg); + return false; /* wait */ + } + + if (dcr->VolCatInfo.VolCatMaxJobs > 0 && (int)dcr->VolCatInfo.VolCatMaxJobs <= + (dev->num_writers + dev->num_reserved())) { /* Max Job Vols depassed or already reserved */ Mmsg(jcr->errmsg, _("3611 JobId=%u Volume max jobs=%d exceeded on %s device %s.\n"), (uint32_t)jcr->JobId, dcr->VolCatInfo.VolCatMaxJobs, @@ -946,21 +1045,28 @@ static int is_pool_ok(DCR *dcr) DEVICE *dev = dcr->dev; JCR *jcr = dcr->jcr; - /* Now check if we want the same Pool and pool type */ - if (strcmp(dev->pool_name, dcr->pool_name) == 0 && - strcmp(dev->pool_type, dcr->pool_type) == 0) { - /* OK, compatible device */ - Dmsg1(dbglvl, "OK dev: %s num_writers=0, reserved, pool matches\n", dev->print_name()); - return 1; - } else { - /* Drive Pool not suitable for us */ - Mmsg(jcr->errmsg, _( -"3608 JobId=%u wants Pool=\"%s\" but have Pool=\"%s\" nreserve=%d on %s device %s.\n"), - (uint32_t)jcr->JobId, dcr->pool_name, dev->pool_name, - dev->num_reserved(), dev->print_type(), dev->print_name()); - Dmsg1(dbglvl, "Failed: %s", jcr->errmsg); - queue_reserve_message(jcr); + if (dev->num_writers >= 0) { + /* Now check if we want the same Pool and pool type */ + if (strcmp(dev->pool_name, dcr->pool_name) == 0 && + strcmp(dev->pool_type, dcr->pool_type) == 0) { + /* OK, compatible device */ + Dmsg1(dbglvl, "OK dev: %s pool matches\n", dev->print_name()); + return 1; + } + } else if (dev->num_reserved() > 0) { + if (strcmp(dev->reserved_pool_name, dcr->pool_name) == 0) { + /* OK, compatible device */ + Dmsg1(dbglvl, "OK dev: %s pool matches\n", dev->print_name()); + return 1; + } } + /* Drive Pool not suitable for us */ + Mmsg(jcr->errmsg, _( +"3608 JobId=%u wants Pool=\"%s\" but have Pool=\"%s\" nreserve=%d on %s device %s.\n"), + (uint32_t)jcr->JobId, dcr->pool_name, dev->pool_name, + dev->num_reserved(), dev->print_type(), dev->print_name()); + Dmsg1(dbglvl, "Failed: %s", jcr->errmsg); + queue_reserve_message(jcr); return 0; } @@ -991,22 +1097,16 @@ static int can_reserve_drive(DCR *dcr, RCTX &rctx) * no unmounted drive is found, we try that drive. This * helps spread the load to the least used drives. */ - if (rctx.try_low_use_drive && dev == rctx.low_use_drive) { + if (rctx.try_low_use_drive && dev == rctx.low_use_drive && + is_pool_ok(dcr)) { Dmsg2(dbglvl, "OK dev=%s == low_drive=%s.\n", dev->print_name(), rctx.low_use_drive->print_name()); + bstrncpy(dev->pool_name, dcr->pool_name, sizeof(dev->pool_name)); + bstrncpy(dev->pool_type, dcr->pool_type, sizeof(dev->pool_type)); return 1; } /* If he wants a free drive, but this one is busy, no go */ if (!rctx.PreferMountedVols && dev->is_busy()) { - /* Save least used drive */ - if ((dev->num_writers + dev->num_reserved()) < rctx.num_writers) { - rctx.num_writers = dev->num_writers + dev->num_reserved(); - rctx.low_use_drive = dev; - Dmsg2(dbglvl, "set low use drive=%s num_writers=%d\n", - dev->print_name(), rctx.num_writers); - } else { - Dmsg1(dbglvl, "not low use num_writers=%d\n", dev->num_writers+dev->num_reserved()); - } Mmsg(jcr->errmsg, _("3605 JobId=%u wants free drive but %s device %s is busy.\n"), jcr->JobId, dev->print_type(), dev->print_name()); queue_reserve_message(jcr); @@ -1054,7 +1154,7 @@ static int can_reserve_drive(DCR *dcr, RCTX &rctx) /* Check for unused autochanger drive */ if (rctx.autochanger_only && !dev->is_busy() && - dev->VolHdr.VolumeName[0] == 0) { + dev->VolHdr.VolumeName[0] == 0 && is_pool_ok(dcr)) { /* Device is available but not yet reserved, reserve it for us */ Dmsg1(dbglvl, "OK Res Unused autochanger %s.\n", dev->print_name()); bstrncpy(dev->pool_name, dcr->pool_name, sizeof(dev->pool_name)); @@ -1090,7 +1190,7 @@ static int can_reserve_drive(DCR *dcr, RCTX &rctx) * Check if the device is in append mode with writers (i.e. * available if pool is the same). */ - if (dev->can_append() || dev->num_writers > 0) { + if (dev->can_append() || dev->num_writers > 0 || dev->num_reserved() > 0) { return is_pool_ok(dcr); } else { Pmsg1(000, _("Logic error!!!! JobId=%u Should not get here.\n"), (int)jcr->JobId); diff --git a/bacula/src/stored/reserve.h b/bacula/src/stored/reserve.h index 4f3d7b305e..bae8d72676 100644 --- a/bacula/src/stored/reserve.h +++ b/bacula/src/stored/reserve.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -50,7 +50,6 @@ public: DIRSTORE *store; DEVRES *device; DEVICE *low_use_drive; /* Low use drive candidate */ - int num_writers; /* for selecting low use drive */ bool try_low_use_drive; /* see if low use drive available */ bool any_drive; /* Accept any drive if set */ bool PreferMountedVols; /* Prefer volumes already mounted */ diff --git a/bacula/src/stored/s3_driver.c b/bacula/src/stored/s3_driver.c new file mode 100644 index 0000000000..c7f424c3e1 --- /dev/null +++ b/bacula/src/stored/s3_driver.c @@ -0,0 +1,806 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Routines for writing to the Cloud using S3 protocol. + * NOTE!!! This cloud driver is not compatible with + * any disk-changer script for changing Volumes. + * It does however work with Bacula Virtual autochangers. + * + * Written by Kern Sibbald, May MMXVI + */ + +#include "s3_driver.h" + +#ifdef HAVE_LIBS3 + +static const int dbglvl = 100; +static const char *S3Errors[] = { + "OK", + "InternalError", + "OutOfMemory", + "Interrupted", + "InvalidBucketNameTooLong", + "InvalidBucketNameFirstCharacter", + "InvalidBucketNameCharacter", + "InvalidBucketNameCharacterSequence", + "InvalidBucketNameTooShort", + "InvalidBucketNameDotQuadNotation", + "QueryParamsTooLong", + "FailedToInitializeRequest", + "MetaDataHeadersTooLong", + "BadMetaData", + "BadContentType", + "ContentTypeTooLong", + "BadMD5", + "MD5TooLong", + "BadCacheControl", + "CacheControlTooLong", + "BadContentDispositionFilename", + "ContentDispositionFilenameTooLong", + "BadContentEncoding", + "ContentEncodingTooLong", + "BadIfMatchETag", + "IfMatchETagTooLong", + "BadIfNotMatchETag", + "IfNotMatchETagTooLong", + "HeadersTooLong", + "KeyTooLong", + "UriTooLong", + "XmlParseFailure", + "EmailAddressTooLong", + "UserIdTooLong", + "UserDisplayNameTooLong", + "GroupUriTooLong", + "PermissionTooLong", + "TargetBucketTooLong", + "TargetPrefixTooLong", + "TooManyGrants", + "BadGrantee", + "BadPermission", + "XmlDocumentTooLarge", + "NameLookupError", + "FailedToConnect", + "ServerFailedVerification", + "ConnectionFailed", + "AbortedByCallback", + "AccessDenied", + "AccountProblem", + "AmbiguousGrantByEmailAddress", + "BadDigest", + "BucketAlreadyExists", + "BucketAlreadyOwnedByYou", + "BucketNotEmpty", + "CredentialsNotSupported", + "CrossLocationLoggingProhibited", + "EntityTooSmall", + "EntityTooLarge", + "ExpiredToken", + "IllegalVersioningConfigurationException", + "IncompleteBody", + "IncorrectNumberOfFilesInPostRequest", + "InlineDataTooLarge", + "InternalError", + "InvalidAccessKeyId", + "InvalidAddressingHeader", + "InvalidArgument", + "InvalidBucketName", + "InvalidBucketState", + "InvalidDigest", + "InvalidLocationConstraint", + "InvalidObjectState", + "InvalidPart", + "InvalidPartOrder", + "InvalidPayer", + "InvalidPolicyDocument", + "InvalidRange", + "InvalidRequest", + "InvalidSecurity", + "InvalidSOAPRequest", + "InvalidStorageClass", + "InvalidTargetBucketForLogging", + "InvalidToken", + "InvalidURI", + "KeyTooLong", + "MalformedACLError", + "MalformedPOSTRequest", + "MalformedXML", + "MaxMessageLengthExceeded", + "MaxPostPreDataLengthExceededError", + "MetadataTooLarge", + "MethodNotAllowed", + "MissingAttachment", + "MissingContentLength", + "MissingRequestBodyError", + "MissingSecurityElement", + "MissingSecurityHeader", + "NoLoggingStatusForKey", + "NoSuchBucket", + "NoSuchKey", + "NoSuchLifecycleConfiguration", + "NoSuchUpload", + "NoSuchVersion", + "NotImplemented", + "NotSignedUp", + "NotSuchBucketPolicy", + "OperationAborted", + "PermanentRedirect", + "PreconditionFailed", + "Redirect", + "RestoreAlreadyInProgress", + "RequestIsNotMultiPartContent", + "RequestTimeout", + "RequestTimeTooSkewed", + "RequestTorrentOfBucketError", + "SignatureDoesNotMatch", + "ServiceUnavailable", + "SlowDown", + "TemporaryRedirect", + "TokenRefreshRequired", + "TooManyBuckets", + "UnexpectedContent", + "UnresolvableGrantByEmailAddress", + "UserKeyMustBeSpecified", + "Unknown", + "HttpErrorMovedTemporarily", + "HttpErrorBadRequest", + "HttpErrorForbidden", + "HttpErrorNotFound", + "HttpErrorConflict", + "HttpErrorUnknown", + "Undefined" +}; + +#define S3ErrorsSize (sizeof(S3Errors)/sizeof(char *)) + +#include + +/* + * Our Bacula context for s3_xxx callbacks + * NOTE: only items needed for particular callback are set + */ +class bacula_ctx { +public: + JCR *jcr; + transfer *xfer; + POOLMEM *&errMsg; + ilist *parts; + int isTruncated; + char* nextMarker; + int64_t obj_len; + const char *caller; + FILE *infile; + FILE *outfile; + alist *volumes; + S3Status status; + bwlimit *limit; /* Used to control the bandwidth */ + bacula_ctx(POOLMEM *&err) : jcr(NULL), xfer(NULL), errMsg(err), parts(NULL), + isTruncated(0), nextMarker(NULL), obj_len(0), caller(NULL), + infile(NULL), outfile(NULL), volumes(NULL), status(S3StatusOK), limit(NULL) + {} + bacula_ctx(transfer *t) : jcr(NULL), xfer(t), errMsg(t->m_message), parts(NULL), + isTruncated(0), nextMarker(NULL), obj_len(0), caller(NULL), + infile(NULL), outfile(NULL), volumes(NULL), status(S3StatusOK), limit(NULL) + {} +}; + + +/* Imported functions */ +const char *mode_to_str(int mode); + +/* Forward referenced functions */ + +/* Const and Static definitions */ + +static S3Status responsePropertiesCallback( + const S3ResponseProperties *properties, + void *callbackData); + +static void responseCompleteCallback( + S3Status status, + const S3ErrorDetails *oops, + void *callbackData); + + +S3ResponseHandler responseHandler = +{ + &responsePropertiesCallback, + &responseCompleteCallback +}; + + + + +static S3Status responsePropertiesCallback( + const S3ResponseProperties *properties, + void *callbackData) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackData; + ASSERT(ctx); + if (ctx->xfer && properties) { + if (properties->contentLength > 0) { + ctx->xfer->m_res_size = properties->contentLength; + } + if (properties->lastModified > 0) { + ctx->xfer->m_res_mtime = properties->lastModified; + } + } + return S3StatusOK; +} + +static void responseCompleteCallback( + S3Status status, + const S3ErrorDetails *oops, + void *callbackCtx) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackCtx; + const char *msg; + + Enter(dbglvl); + if (ctx) { + ctx->status = status; /* return completion status */ + } + if (status < 0 || status > S3ErrorsSize) { + status = (S3Status)S3ErrorsSize; + } + msg = oops->message; + if (!msg) { + msg = S3Errors[status]; + } + if ((status != S3StatusOK) && ctx->errMsg) { + if (oops->furtherDetails) { + Mmsg(ctx->errMsg, "%s ERR=%s\n" + "furtherDetails=%s\n", ctx->caller, msg, oops->furtherDetails); + Dmsg1(dbglvl, "%s", ctx->errMsg); + } else { + Mmsg(ctx->errMsg, "%s ERR=%s\n", ctx->caller, msg); + Dmsg1(dbglvl, "%s", ctx->errMsg); + } + } + return; +} + + + + +static int putObjectCallback(int buf_len, char *buf, void *callbackCtx) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackCtx; + + ssize_t rbytes = 0; + int read_len; + + if (ctx->xfer->is_cancelled()) { + Mmsg(ctx->errMsg, _("Job cancelled.\n")); + return -1; + } + if (ctx->obj_len) { + read_len = (ctx->obj_len > buf_len) ? buf_len : ctx->obj_len; + rbytes = fread(buf, 1, read_len, ctx->infile); + Dmsg5(dbglvl, "%s thread=%lu rbytes=%d bufsize=%u remlen=%lu\n", + ctx->caller, pthread_self(), rbytes, buf_len, ctx->obj_len); + if (rbytes <= 0) { + berrno be; + Mmsg(ctx->errMsg, "%s Error reading input file: ERR=%s\n", + ctx->caller, be.bstrerror()); + goto get_out; + } + ctx->obj_len -= rbytes; + + if (ctx->limit) { + ctx->limit->control_bwlimit(rbytes); + } + } + +get_out: + return rbytes; +} + +S3PutObjectHandler putObjectHandler = +{ + responseHandler, + &putObjectCallback +}; + + +/* + * Put a cache object into the cloud + */ +S3Status s3_driver::put_object(transfer *xfer, const char *cache_fname, const char *cloud_fname) +{ + Enter(dbglvl); + bacula_ctx ctx(xfer); + ctx.limit = upload_limit.use_bwlimit() ? &upload_limit : NULL; + + struct stat statbuf; + if (lstat(cache_fname, &statbuf) == -1) { + berrno be; + Mmsg2(ctx.errMsg, "Failed to stat file %s. ERR=%s\n", + cache_fname, be.bstrerror()); + goto get_out; + } + + ctx.obj_len = statbuf.st_size; + + if (!(ctx.infile = bfopen(cache_fname, "r"))) { + berrno be; + Mmsg2(ctx.errMsg, "Failed to open input file %s. ERR=%s\n", + cache_fname, be.bstrerror()); + goto get_out; + } + + ctx.caller = "S3_put_object"; + S3_put_object(&s3ctx, cloud_fname, ctx.obj_len, NULL, NULL, + &putObjectHandler, &ctx); + +get_out: + if (ctx.infile) { + fclose(ctx.infile); + } + + /* no error so far -> retrieve uploaded part info */ + if (ctx.errMsg[0] == 0) { + ilist parts; + get_cloud_volume_parts_list(xfer->m_dcr, cloud_fname, &parts, ctx.errMsg); + for (int i=1; i <= parts.last_index() ; i++) { + cloud_part *p = (cloud_part *)parts.get(i); + if (p) { + xfer->m_res_size = p->size; + xfer->m_res_mtime = p->mtime; + break; /* not need to go further */ + } + } + } + + return ctx.status; +} + +static S3Status getObjectDataCallback(int buf_len, const char *buf, + void *callbackCtx) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackCtx; + ssize_t wbytes; + + Enter(dbglvl); + if (ctx->xfer->is_cancelled()) { + Mmsg(ctx->errMsg, _("Job cancelled.\n")); + return S3StatusAbortedByCallback; + } + /* Write buffer to output file */ + wbytes = fwrite(buf, 1, buf_len, ctx->outfile); + if (wbytes < 0) { + berrno be; + Mmsg(ctx->errMsg, "%s Error writing output file: ERR=%s\n", + ctx->caller, be.bstrerror()); + return S3StatusAbortedByCallback; + } + + if (ctx->limit) { + ctx->limit->control_bwlimit(wbytes); + } + return ((wbytes < buf_len) ? + S3StatusAbortedByCallback : S3StatusOK); +} + + +bool s3_driver::get_cloud_object(transfer *xfer, const char *cloud_fname, const char *cache_fname) +{ + int64_t ifModifiedSince = -1; + int64_t ifNotModifiedSince = -1; + const char *ifMatch = 0; + const char *ifNotMatch = 0; + uint64_t startByte = 0; + uint64_t byteCount = 0; + bacula_ctx ctx(xfer); + ctx.limit = download_limit.use_bwlimit() ? &download_limit : NULL; + + Enter(dbglvl); + /* Initialize handlers */ + S3GetConditions getConditions = { + ifModifiedSince, + ifNotModifiedSince, + ifMatch, + ifNotMatch + }; + S3GetObjectHandler getObjectHandler = { + { &responsePropertiesCallback, &responseCompleteCallback }, + &getObjectDataCallback + }; + + + /* see if cache file already exists */ + struct stat buf; + if (lstat(cache_fname, &buf) == -1) { + ctx.outfile = bfopen(cache_fname, "w"); + } else { + /* Exists so truncate and write from beginning */ + ctx.outfile = bfopen(cache_fname, "r+"); + } + + if (!ctx.outfile) { + berrno be; + Mmsg2(ctx.errMsg, "Could not open cache file %s. ERR=%s\n", + cache_fname, be.bstrerror()); + goto get_out; + } + + + ctx.caller = "S3_get_object"; + S3_get_object(&s3ctx, cloud_fname, &getConditions, startByte, + byteCount, 0, &getObjectHandler, &ctx); + + if (fclose(ctx.outfile) < 0) { + berrno be; + Mmsg2(ctx.errMsg, "Error closing cache file %s: %s\n", + cache_fname, be.bstrerror()); + } + +get_out: + return (ctx.errMsg[0] == 0); +} + +/* + * Not thread safe + */ +bool s3_driver::truncate_cloud_volume(DCR *dcr, const char *VolumeName, ilist *trunc_parts, POOLMEM *&err) +{ + Enter(dbglvl); + + bacula_ctx ctx(err); + ctx.jcr = dcr->jcr; + + int last_index = (int)trunc_parts->last_index(); + POOLMEM *cloud_fname = get_pool_memory(PM_FNAME); + for (int i=1; (i<=last_index); i++) { + if (!trunc_parts->get(i)) { + continue; + } + if (ctx.jcr->is_canceled()) { + Mmsg(err, _("Job cancelled.\n")); + goto get_out; + } + /* don't forget to specify the volume name is the object path */ + make_cloud_filename(cloud_fname, VolumeName, i); + Dmsg1(dbglvl, "Object to truncate: %s\n", cloud_fname); + ctx.caller = "S3_delete_object"; + S3_delete_object(&s3ctx, cloud_fname, 0, &responseHandler, &ctx); + if (ctx.status != S3StatusOK) { + /* error message should have been filled within response cb */ + goto get_out; + } + } + +get_out: + free_pool_memory(cloud_fname); + bfree_and_null(ctx.nextMarker); + return (err[0] == 0); +} + +void s3_driver::make_cloud_filename(POOLMEM *&filename, + const char *VolumeName, uint32_t apart) +{ + Enter(dbglvl); + filename[0] = 0; + dev->add_vol_and_part(filename, VolumeName, "part", apart); + Dmsg1(dbglvl, "make_cloud_filename: %s\n", filename); +} + +bool s3_driver::retry_put_object(S3Status status) +{ + return ( + status == S3StatusFailedToConnect || + status == S3StatusConnectionFailed + ); +} + +/* + * Copy a single cache part to the cloud + */ +bool s3_driver::copy_cache_part_to_cloud(transfer *xfer) +{ + Enter(dbglvl); + POOLMEM *cloud_fname = get_pool_memory(PM_FNAME); + make_cloud_filename(cloud_fname, xfer->m_volume_name, xfer->m_part); + uint32_t retry = max_upload_retries; + S3Status status = S3StatusOK; + do { + status = put_object(xfer, xfer->m_cache_fname, cloud_fname); + --retry; + } while (retry_put_object(status) && (retry>0)); + free_pool_memory(cloud_fname); + return (status == S3StatusOK); +} + +/* + * Copy a single object (part) from the cloud to the cache + */ +bool s3_driver::copy_cloud_part_to_cache(transfer *xfer) +{ + Enter(dbglvl); + POOLMEM *cloud_fname = get_pool_memory(PM_FNAME); + make_cloud_filename(cloud_fname, xfer->m_volume_name, xfer->m_part); + bool rtn = get_cloud_object(xfer, cloud_fname, xfer->m_cache_fname); + free_pool_memory(cloud_fname); + return rtn; +} + +/* + * NOTE: See the SD Cloud resource in stored_conf.h +*/ + +bool s3_driver::init(JCR *jcr, cloud_dev *adev, DEVRES *adevice) +{ + S3Status status; + + dev = adev; /* copy cloud device pointer */ + device = adevice; /* copy device resource pointer */ + cloud = device->cloud; /* local pointer to cloud definition */ + + /* Setup bucket context for S3 lib */ + s3ctx.hostName = cloud->host_name; + s3ctx.bucketName = cloud->bucket_name; + s3ctx.protocol = (S3Protocol)cloud->protocol; + s3ctx.uriStyle = (S3UriStyle)cloud->uri_style; + s3ctx.accessKeyId = cloud->access_key; + s3ctx.secretAccessKey = cloud->secret_key; + s3ctx.authRegion = cloud->region; + + /* File I/O buffer */ + buf_len = dev->max_block_size; + if (buf_len == 0) { + buf_len = DEFAULT_BLOCK_SIZE; + } + + if ((status = S3_initialize("s3", S3_INIT_ALL, s3ctx.hostName)) != S3StatusOK) { + Mmsg1(dev->errmsg, "Failed to initialize S3 lib. ERR=%s\n", S3_get_status_name(status)); + Qmsg1(jcr, M_FATAL, 0, "%s", dev->errmsg); + Tmsg1(0, "%s", dev->errmsg); + return false; + } + return true; +} + +bool s3_driver::start_of_job(DCR *dcr) +{ + Jmsg(dcr->jcr, M_INFO, 0, _("Using S3 cloud driver Host=%s Bucket=%s\n"), + s3ctx.hostName, s3ctx.bucketName); + return true; +} + +bool s3_driver::end_of_job(DCR *dcr) +{ + return true; +} + +/* + * Note, dcr may be NULL + */ +bool s3_driver::term(DCR *dcr) +{ + S3_deinitialize(); + return true; +} + + + +/* + * libs3 callback for get_cloud_volume_parts_list() + */ +static S3Status partslistBucketCallback( + int isTruncated, + const char *nextMarker, + int numObj, + const S3ListBucketContent *object, + int commonPrefixesCount, + const char **commonPrefixes, + void *callbackCtx) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackCtx; + + Enter(dbglvl); + for (int i = 0; ctx->parts && (i < numObj); i++) { + const S3ListBucketContent *obj = &(object[i]); + const char *ext=strstr(obj->key, "part."); + if (obj && ext!=NULL) { + cloud_part *part = (cloud_part*) malloc(sizeof(cloud_part)); + + part->index = atoi(&(ext[5])); + part->mtime = obj->lastModified; + part->size = obj->size; + ctx->parts->put(part->index, part); + } + } + + ctx->isTruncated = isTruncated; + if (ctx->nextMarker) { + bfree_and_null(ctx->nextMarker); + } + if (nextMarker) { + ctx->nextMarker = bstrdup(nextMarker); + } + + Leave(dbglvl); + if (ctx->jcr->is_canceled()) { + Mmsg(ctx->errMsg, _("Job cancelled.\n")); + return S3StatusAbortedByCallback; + } + return S3StatusOK; +} + +S3ListBucketHandler partslistBucketHandler = +{ + responseHandler, + &partslistBucketCallback +}; + +bool s3_driver::get_cloud_volume_parts_list(DCR *dcr, const char* VolumeName, ilist *parts, POOLMEM *&err) +{ + JCR *jcr = dcr->jcr; + Enter(dbglvl); + + if (!parts || strlen(VolumeName) == 0) { + pm_strcpy(err, "Invalid argument"); + return false; + } + + bacula_ctx ctx(err); + ctx.jcr = jcr; + ctx.parts = parts; + ctx.isTruncated = 1; /* pass into the while loop at least once */ + ctx.caller = "S3_list_bucket"; + while (ctx.isTruncated!=0) { + ctx.isTruncated = 0; + S3_list_bucket(&s3ctx, VolumeName, ctx.nextMarker, NULL, 0, NULL, + &partslistBucketHandler, &ctx); + if (ctx.status != S3StatusOK) { + pm_strcpy(err, S3Errors[ctx.status]); + bfree_and_null(ctx.nextMarker); + return false; + } + } + bfree_and_null(ctx.nextMarker); + return true; + +} + +/* + * libs3 callback for get_cloud_volumes_list() + */ +static S3Status volumeslistBucketCallback( + int isTruncated, + const char *nextMarker, + int numObj, + const S3ListBucketContent *object, + int commonPrefixesCount, + const char **commonPrefixes, + void *callbackCtx) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackCtx; + + Enter(dbglvl); + for (int i = 0; ctx->volumes && (i < commonPrefixesCount); i++) { + char *cp = bstrdup(commonPrefixes[i]); + cp[strlen(cp)-1] = 0; + ctx->volumes->append(cp); + } + + ctx->isTruncated = isTruncated; + if (ctx->nextMarker) { + bfree_and_null(ctx->nextMarker); + } + if (nextMarker) { + ctx->nextMarker = bstrdup(nextMarker); + } + + Leave(dbglvl); + if (ctx->jcr->is_canceled()) { + Mmsg(ctx->errMsg, _("Job cancelled.\n")); + return S3StatusAbortedByCallback; + } + return S3StatusOK; +} + +S3ListBucketHandler volumeslistBucketHandler = +{ + responseHandler, + &volumeslistBucketCallback +}; + +bool s3_driver::get_cloud_volumes_list(DCR *dcr, alist *volumes, POOLMEM *&err) +{ + JCR *jcr = dcr->jcr; + Enter(dbglvl); + + if (!volumes) { + pm_strcpy(err, "Invalid argument"); + return false; + } + + bacula_ctx ctx(err); + ctx.volumes = volumes; + ctx.jcr = jcr; + ctx.isTruncated = 1; /* pass into the while loop at least once */ + ctx.caller = "S3_list_bucket"; + while (ctx.isTruncated!=0) { + ctx.isTruncated = 0; + S3_list_bucket(&s3ctx, NULL, ctx.nextMarker, "/", 0, NULL, + &volumeslistBucketHandler, &ctx); + if (ctx.status != S3StatusOK) { + break; + } + } + bfree_and_null(ctx.nextMarker); + return (err[0] == 0); +} + +#ifdef really_needed +static S3Status listBucketCallback( + int isTruncated, + const char *nextMarker, + int contentsCount, + const S3ListBucketContent *contents, + int commonPrefixesCount, + const char **commonPrefixes, + void *callbackData); + +S3ListBucketHandler listBucketHandler = +{ + responseHandler, + &listBucketCallback +}; + + +/* + * List content of a bucket + */ +static S3Status listBucketCallback( + int isTruncated, + const char *nextMarker, + int numObj, + const S3ListBucketContent *contents, + int commonPrefixesCount, + const char **commonPrefixes, + void *callbackData) +{ + bacula_ctx *ctx = (bacula_ctx *)callbackCtx; + if (print_hdr) { + Pmsg1(000, "\n%-22s", " Object Name"); + Pmsg2(000, " %-5s %-20s", "Size", " Last Modified"); + Pmsg0(000, "\n---------------------- ----- --------------------\n"); + print_hdr = false; /* print header once only */ + } + + for (int i = 0; i < numObj; i++) { + char timebuf[256]; + char sizebuf[16]; + const S3ListBucketContent *content = &(contents[i]); + time_t t = (time_t) content->lastModified; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); + sprintf(sizebuf, "%5llu", (unsigned long long) content->size); + Pmsg3(000, "%-22s %s %s\n", content->key, sizebuf, timebuf); + } + Pmsg0(000, "\n"); + if (ctx->jcr->is_canceled()) { + Mmsg(ctx->errMsg, _("Job cancelled.\n")); + return S3StatusAbortedByCallback; + } + return S3StatusOK; +} +#endif + +#endif /* HAVE_LIBS3 */ diff --git a/bacula/src/stored/s3_driver.h b/bacula/src/stored/s3_driver.h new file mode 100644 index 0000000000..016b6d6ec3 --- /dev/null +++ b/bacula/src/stored/s3_driver.h @@ -0,0 +1,67 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * Routines for writing to the Cloud using S3 protocol. + * + * Written by Kern Sibbald, May MMXVI + */ + +#ifndef _S3_DRV_H +#define _S3_DRV_H + +#include "bacula.h" +#include "stored.h" + +#ifdef HAVE_LIBS3 +#include +#include "cloud_driver.h" /* get base class definitions */ + +class s3_driver: public cloud_driver { +private: + S3BucketContext s3ctx; /* Main S3 bucket context */ + uint32_t buf_len; + +public: + cloud_dev *dev; /* device that is calling us */ + DEVRES *device; + DCR *dcr; /* dcr set during calls to S3_xxx */ + CLOUD *cloud; /* Pointer to CLOUD resource */ + + s3_driver() { + }; + ~s3_driver() { + }; + + void make_cloud_filename(POOLMEM *&filename, const char *VolumeName, uint32_t part); + bool init(JCR *jcr, cloud_dev *dev, DEVRES *device); + bool start_of_job(DCR *dcr); + bool term(DCR *dcr); + bool end_of_job(DCR *dcr); + bool truncate_cloud_volume(DCR *dcr, const char *VolumeName, ilist *trunc_parts, POOLMEM *&err); + bool copy_cache_part_to_cloud(transfer *xfer); + bool copy_cloud_part_to_cache(transfer *xfer); + bool get_cloud_volume_parts_list(DCR *dcr, const char* VolumeName, ilist *parts, POOLMEM *&err); + bool get_cloud_volumes_list(DCR* dcr, alist *volumes, POOLMEM *&err); + S3Status put_object(transfer *xfer, const char *cache_fname, const char *cloud_fname); + bool retry_put_object(S3Status status); + bool get_cloud_object(transfer *xfer, const char *cloud_fname, const char *cache_fname); +}; + +#endif /* HAVE_LIBS3 */ +#endif /* _S3_DRV_H */ diff --git a/bacula/src/stored/scan.c b/bacula/src/stored/scan.c index d5afc0e80b..60a28eda2a 100644 --- a/bacula/src/stored/scan.c +++ b/bacula/src/stored/scan.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -105,9 +105,9 @@ bool DEVICE::scan_dir_for_volume(DCR *dcr) * this volume is really OK. If not, put back the desired * volume name, mark it not in changer and continue. */ - /* Check if this is a valid Volume in the pool */ bstrncpy(dcr->VolumeName, result->d_name, sizeof(dcr->VolumeName)); - if (!dir_get_volume_info(dcr, GET_VOL_INFO_FOR_WRITE)) { + /* Check if this is a valid Volume in the pool */ + if (!dir_get_volume_info(dcr, dcr->VolumeName, GET_VOL_INFO_FOR_WRITE)) { continue; } /* This was not the volume we expected, but it is OK with diff --git a/bacula/src/stored/sd_plugins.c b/bacula/src/stored/sd_plugins.c index b8550f2550..ada4c26e6c 100644 --- a/bacula/src/stored/sd_plugins.c +++ b/bacula/src/stored/sd_plugins.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -100,14 +100,14 @@ int generate_plugin_event(JCR *jcr, bsdEventType eventType, void *value) if (!b_plugin_list) { Dmsg0(dbglvl, "No b_plugin_list: generate_plugin_event ignored.\n"); - return bRC_OK; - } - if (!jcr) { - Dmsg0(dbglvl, "jcr==NULL: generate_plugin_event ignored.\n"); - return bRC_OK; - } - if (!jcr->plugin_ctx_list) { - Dmsg0(dbglvl, "No plugin_ctx_list: generate_plugin_event ignored.\n"); + return bRC_OK; + } + if (!jcr) { + Dmsg0(dbglvl, "No jcr: generate_plugin_event ignored.\n"); + return bRC_OK; + } + if (!jcr->plugin_ctx_list) { + Dmsg0(dbglvl, "No plugin_ctx_list: generate_plugin_event ignored.\n"); return bRC_OK; /* Return if no plugins loaded */ } @@ -288,9 +288,12 @@ void new_plugins(JCR *jcr) if (jcr->is_job_canceled()) { return; } - - /* If plugins already loaded, just return */ - if (jcr->plugin_ctx_list) return; + /* + * If plugins already loaded, just return + */ + if (jcr->plugin_ctx_list) { + return; + } int num = b_plugin_list->size(); @@ -347,7 +350,6 @@ void free_plugins(JCR *jcr) * ============================================================== */ - static bRC baculaGetValue(bpContext *ctx, bsdrVariable var, void *value) { JCR *jcr; @@ -370,6 +372,10 @@ static bRC baculaGetValue(bpContext *ctx, bsdrVariable var, void *value) *((char **)value) = jcr->Job; Dmsg1(dbglvl, "Bacula: return Job name=%s\n", jcr->Job); break; +#if 0 + case bsdVarDevTypes: + *((s_kw **)value) = dev_types; +#endif default: break; } diff --git a/bacula/src/stored/spool.c b/bacula/src/stored/spool.c index db31745d4e..cfb16f85d3 100644 --- a/bacula/src/stored/spool.c +++ b/bacula/src/stored/spool.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -34,8 +34,9 @@ static bool despool_data(DCR *dcr, bool commit); static int read_block_from_spool_file(DCR *dcr); static bool open_attr_spool_file(JCR *jcr, BSOCK *bs); static bool close_attr_spool_file(JCR *jcr, BSOCK *bs); -static bool write_spool_header(DCR *dcr); -static bool write_spool_data(DCR *dcr); +static ssize_t write_spool_header(DCR *dcr, ssize_t *expected); +static ssize_t write_spool_data(DCR *dcr, ssize_t *expected); +static bool write_spool_block(DCR *dcr); struct spool_stats_t { uint32_t data_jobs; /* current jobs spooling data */ @@ -94,7 +95,10 @@ void list_spool_stats(void sendit(const char *msg, int len, void *sarg), void *a bool begin_data_spool(DCR *dcr) { bool stat = true; - if (!dcr->dev->is_dvd() && dcr->jcr->spool_data) { + if (dcr->dev->is_aligned()) { + dcr->jcr->spool_data = false; + } + if (dcr->jcr->spool_data) { Dmsg0(100, "Turning on data spooling\n"); dcr->spool_data = true; stat = open_data_spool_file(dcr); @@ -154,7 +158,7 @@ static bool open_data_spool_file(DCR *dcr) int spool_fd; make_unique_data_spool_filename(dcr, &name); - if ((spool_fd = open(name, O_CREAT|O_TRUNC|O_RDWR|O_BINARY, 0640)) >= 0) { + if ((spool_fd = open(name, O_CREAT|O_TRUNC|O_RDWR|O_BINARY|O_CLOEXEC, 0640)) >= 0) { dcr->spool_fd = spool_fd; dcr->jcr->spool_attributes = true; } else { @@ -222,8 +226,7 @@ static bool despool_data(DCR *dcr, bool commit) * We create a dev structure to read from the spool file * in rdev and rdcr. */ - rdev = (DEVICE *)malloc(sizeof(DEVICE)); - memset(rdev, 0, sizeof(DEVICE)); + rdev = New(file_dev); rdev->dev_name = get_memory(strlen(spool_name)+1); bstrncpy(rdev->dev_name, spool_name, strlen(spool_name)+1); rdev->errmsg = get_pool_memory(PM_EMSG); @@ -249,10 +252,6 @@ static bool despool_data(DCR *dcr, bool commit) set_new_file_parameters(dcr); for ( ; ok; ) { - if (job_canceled(jcr)) { - ok = false; - break; - } stat = read_block_from_spool_file(rdcr); if (stat == RB_EOT) { break; @@ -261,6 +260,11 @@ static bool despool_data(DCR *dcr, bool commit) break; } ok = dcr->write_block_to_device(); + + if (jcr->is_canceled()) { + ok = false; + break; + } if (!ok) { Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), dcr->dev->print_name(), dcr->dev->bstrerror()); @@ -390,6 +394,7 @@ static int read_block_from_spool_file(DCR *dcr) block->LastIndex = hdr.LastIndex; block->VolSessionId = dcr->jcr->VolSessionId; block->VolSessionTime = dcr->jcr->VolSessionTime; + Dmsg2(800, "Read block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex); return RB_OK; } @@ -456,120 +461,112 @@ bool write_block_to_spool_file(DCR *dcr) Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data again ...\n")); } - - if (!write_spool_header(dcr)) { + if (!write_spool_block(dcr)) { return false; } - if (!write_spool_data(dcr)) { - return false; - } Dmsg2(800, "Wrote block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex); empty_block(block); return true; } -static bool write_spool_header(DCR *dcr) +static bool rewind_spoolfile(DCR *dcr, ssize_t size, ssize_t expected) { - spool_hdr hdr; - ssize_t stat; - DEV_BLOCK *block = dcr->block; JCR *jcr = dcr->jcr; - - hdr.FirstIndex = block->FirstIndex; - hdr.LastIndex = block->LastIndex; - hdr.len = block->binbuf; - - /* Write header */ - for (int retry=0; retry<=1; retry++) { - stat = write(dcr->spool_fd, (char*)&hdr, sizeof(hdr)); - if (stat == -1) { - berrno be; - Jmsg(jcr, M_FATAL, 0, _("Error writing header to spool file. ERR=%s\n"), - be.bstrerror()); - jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ - } - if (stat != (ssize_t)sizeof(hdr)) { - Jmsg(jcr, M_ERROR, 0, _("Error writing header to spool file." - " Disk probably full. Attempting recovery. Wanted to write=%d got=%d\n"), - (int)stat, (int)sizeof(hdr)); - /* If we wrote something, truncate it, then despool */ - if (stat != -1) { + if (size == 0) { + return true; /* nothing to do */ + } + Jmsg(jcr, M_ERROR, 0, _("Error writing header to spool file." + " Disk probably full. Attempting recovery. Wanted to write=%d got=%d\n"), + (int)expected, (int)size); #if defined(HAVE_WIN32) - boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR); + boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR); #else - boffset_t pos = lseek(dcr->spool_fd, 0, SEEK_CUR); + boffset_t pos = lseek(dcr->spool_fd, 0, SEEK_CUR); #endif - if (ftruncate(dcr->spool_fd, pos - stat) != 0) { - berrno be; - Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), - be.bstrerror()); - /* Note, try continuing despite ftruncate problem */ - } - } - if (!despool_data(dcr, false)) { - Jmsg(jcr, M_FATAL, 0, _("Fatal despooling error.")); - jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ - return false; - } - continue; /* try again */ - } - return true; + if (ftruncate(dcr->spool_fd, pos - size) != 0) { + berrno be; + Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), + be.bstrerror()); + /* Note, try continuing despite ftruncate problem */ } - Jmsg(jcr, M_FATAL, 0, _("Retrying after header spooling error failed.\n")); - jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ - return false; + if (!despool_data(dcr, false)) { + Jmsg(jcr, M_FATAL, 0, _("Fatal despooling error.")); + jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ + return false; + } + return true; } -static bool write_spool_data(DCR *dcr) +static bool write_spool_block(DCR *dcr) { - ssize_t stat; - DEV_BLOCK *block = dcr->block; - JCR *jcr = dcr->jcr; + ssize_t size = 0, ret; + ssize_t expected = 0; - /* Write data */ - for (int retry=0; retry<=1; retry++) { - stat = write(dcr->spool_fd, block->buf, (size_t)block->binbuf); - if (stat == -1) { - berrno be; - Jmsg(jcr, M_FATAL, 0, _("Error writing data to spool file. ERR=%s\n"), - be.bstrerror()); - jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ + for (int retry=0; retry <= 1; retry++) { + /* Rewind if needed */ + if (size > 0 && !rewind_spoolfile(dcr, size, expected)) { + return false; } - if (stat != (ssize_t)block->binbuf) { - /* - * If we wrote something, truncate it and the header, then despool - */ - if (stat != -1) { -#if defined(HAVE_WIN32) - boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR); -#else - boffset_t pos = lseek(dcr->spool_fd, 0, SEEK_CUR); -#endif - if (ftruncate(dcr->spool_fd, pos - stat - sizeof(spool_hdr)) != 0) { - berrno be; - Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), - be.bstrerror()); - /* Note, try continuing despite ftruncate problem */ - } - } - if (!despool_data(dcr, false)) { - Jmsg(jcr, M_FATAL, 0, _("Fatal despooling error.")); - jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ - return false; - } - if (!write_spool_header(dcr)) { - return false; - } - continue; /* try again */ + + /* Try to write the header */ + ret = write_spool_header(dcr, &expected); + if (ret == -1) { /* I/O error, it's fatal */ + goto bail_out; + + } else { + size += ret; /* Keep the size written for a future rewind */ + } + + if (ret != expected) { /* We don't have the size expected, rewind, despool and retry */ + continue; + } + + ret = write_spool_data(dcr, &expected); + if (ret == -1) { /* I/O Error, it's fatal */ + goto bail_out; + + } else { + size += ret; /* Keep the size written for a furture rewind */ + } + + if (ret != expected) { /* We don't have the size expected, rewind, despool and retry */ + continue; } + return true; } - Jmsg(jcr, M_FATAL, 0, _("Retrying after data spooling error failed.\n")); - jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ + +bail_out: + berrno be; + Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing block to spool file. ERR=%s\n"), + be.bstrerror()); + dcr->jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ return false; } +static ssize_t write_spool_header(DCR *dcr, ssize_t *expected) +{ + spool_hdr hdr; + DEV_BLOCK *block = dcr->block; + + hdr.FirstIndex = block->FirstIndex; + hdr.LastIndex = block->LastIndex; + hdr.len = block->binbuf; + *expected = sizeof(hdr); + + /* Write header */ + return write(dcr->spool_fd, (char*)&hdr, sizeof(hdr)); +} + + +static ssize_t write_spool_data(DCR *dcr, ssize_t *expected) +{ + DEV_BLOCK *block = dcr->block; + *expected = block->binbuf; + return write(dcr->spool_fd, block->buf, (size_t)block->binbuf); +} + static bool close_data_spool_file(DCR *dcr) { POOLMEM *name = get_pool_memory(PM_MESSAGE); @@ -648,7 +645,7 @@ static bool blast_attr_spool_file(JCR *jcr, boffset_t size) POOLMEM *name = get_pool_memory(PM_MESSAGE); make_unique_spool_filename(jcr, &name, jcr->dir_bsock->m_fd); bash_spaces(name); - jcr->dir_bsock->fsend("BlastAttr Job=%s File=%s\n", jcr->Job, name); + jcr->dir_bsock->fsend("BlastAttr JobId=%d File=%s\n", jcr->JobId, name); free_pool_memory(name); if (jcr->dir_bsock->recv() <= 0) { @@ -735,7 +732,7 @@ static bool open_attr_spool_file(JCR *jcr, BSOCK *bs) POOLMEM *name = get_pool_memory(PM_MESSAGE); make_unique_spool_filename(jcr, &name, bs->m_fd); - bs->m_spool_fd = fopen(name, "w+b"); + bs->m_spool_fd = bfopen(name, "w+b"); if (!bs->m_spool_fd) { berrno be; Jmsg(jcr, M_FATAL, 0, _("fopen attr spool file %s failed: ERR=%s\n"), name, diff --git a/bacula/src/stored/status.c b/bacula/src/stored/status.c index cac05c8077..28ee928026 100644 --- a/bacula/src/stored/status.c +++ b/bacula/src/stored/status.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -52,6 +52,40 @@ static void list_jobs_waiting_on_reservation(STATUS_PKT *sp); static void list_status_header(STATUS_PKT *sp); static void list_devices(STATUS_PKT *sp, char *name=NULL); static void list_plugins(STATUS_PKT *sp); +static void list_cloud_transfers(STATUS_PKT *sp, bool verbose); + +void status_alert_callback(void *ctx, const char *short_msg, + const char *long_msg, char *Volume, int severity, + int flags, int alertno, utime_t alert_time) +{ + STATUS_PKT *sp = (STATUS_PKT *)ctx; + const char *type = "Unknown"; + POOL_MEM send_msg(PM_MESSAGE); + char edt[50]; + int len; + + switch (severity) { + case 'C': + type = "Critical"; + break; + case 'W': + type = "Warning"; + break; + case 'I': + type = "Info"; + break; + } + bstrftimes(edt, sizeof(edt), alert_time); + if (chk_dbglvl(10)) { + len = Mmsg(send_msg, _(" %s Alert: at %s Volume=\"%s\" flags=0x%x alert=%s\n"), + type, edt, Volume, flags, long_msg); + } else { + len = Mmsg(send_msg, _(" %s Alert: at %s Volume=\"%s\" alert=%s\n"), + type, edt, Volume, short_msg); + } + sendit(send_msg, len, sp); +} + /* * Status command from Director @@ -83,6 +117,11 @@ void output_status(STATUS_PKT *sp) */ list_devices(sp); + /* + * List cloud transfers + */ + list_cloud_transfers(sp, false); + len = Mmsg(msg, _("Used Volume status:\n")); if (!sp->api) sendit(msg, len, sp); @@ -132,6 +171,117 @@ static find_device(char *devname) } #endif +static void api_list_one_device(char *name, DEVICE *dev, STATUS_PKT *sp) +{ + OutputWriter ow(sp->api_opts); + int zero=0; + int blocked=0; + uint64_t f, t; + const char *p=NULL; + + if (!dev) { + return; + } + + dev->get_freespace(&f, &t); + + ow.get_output(OT_START_OBJ, + OT_STRING, "name", dev->device->hdr.name, + OT_STRING, "archive_device", dev->archive_name(), + OT_STRING, "type", dev->print_type(), + OT_STRING, "media_type", dev->device->media_type, + OT_INT, "open", (int)dev->is_open(), + OT_INT, "writers", dev->num_writers, + OT_INT32, "maximum_concurrent_jobs", dev->max_concurrent_jobs, + OT_INT64, "maximum_volume_size", dev->max_volume_size, + OT_INT, "read_only", dev->device->read_only, + OT_INT, "autoselect", dev->device->autoselect, + OT_INT, "enabled", dev->enabled, + OT_INT64, "free_space", f, + OT_INT64, "total_space", t, + OT_INT64, "devno", dev->devno, + OT_END); + + if (dev->is_open()) { + if (dev->is_labeled()) { + ow.get_output(OT_STRING, "mounted", dev->blocked()?"0":"1", + OT_STRING, "waiting", dev->blocked()?"1":"0", + OT_STRING, "volume", dev->VolHdr.VolumeName, + OT_STRING, "pool", NPRTB(dev->pool_name), + OT_END); + } else { + ow.get_output(OT_INT, "mounted", zero, + OT_INT, "waiting", zero, + OT_STRING, "volume", "", + OT_STRING, "pool", "", + OT_END); + } + + blocked = 1; + switch(dev->blocked()) { + case BST_UNMOUNTED: + p = "User unmounted"; + break; + case BST_UNMOUNTED_WAITING_FOR_SYSOP: + p = "User unmounted during wait for media/mount"; + break; + case BST_DOING_ACQUIRE: + p = "Device is being initialized"; + break; + case BST_WAITING_FOR_SYSOP: + p = "Waiting for mount or create a volume"; + break; + case BST_WRITING_LABEL: + p = "Labeling a Volume"; + break; + default: + blocked=0; + p = NULL; + } + + /* TODO: give more information about blocked status + * and the volume needed if WAITING for SYSOP + */ + ow.get_output(OT_STRING, "blocked_desc", NPRTB(p), + OT_INT, "blocked", blocked, + OT_END); + + ow.get_output(OT_INT, "append", (int)dev->can_append(), + OT_END); + + if (dev->can_append()) { + ow.get_output(OT_INT64, "bytes", dev->VolCatInfo.VolCatBytes, + OT_INT32, "blocks", dev->VolCatInfo.VolCatBlocks, + OT_END); + + } else { /* reading */ + ow.get_output(OT_INT64, "bytes", dev->VolCatInfo.VolCatRBytes, + OT_INT32, "blocks", dev->VolCatInfo.VolCatReads, /* might not be blocks */ + OT_END); + + } + ow.get_output(OT_INT, "file", dev->file, + OT_INT, "block", dev->block_num, + OT_END); + } else { + ow.get_output(OT_INT, "mounted", zero, + OT_INT, "waiting", zero, + OT_STRING, "volume", "", + OT_STRING, "pool", "", + OT_STRING, "blocked_desc", "", + OT_INT, "blocked", zero, + OT_INT, "append", zero, + OT_INT, "bytes", zero, + OT_INT, "blocks", zero, + OT_INT, "file", zero, + OT_INT, "block", zero, + OT_END); + } + + p = ow.get_output(OT_END_OBJ, OT_END); + sendit(p, strlen(p), sp); +} + static void list_one_device(char *name, DEVICE *dev, STATUS_PKT *sp) { @@ -140,6 +290,11 @@ static void list_one_device(char *name, DEVICE *dev, STATUS_PKT *sp) int len; int bpb; + if (sp->api > 1) { + api_list_one_device(name, dev, sp); + return; + } + if (!dev) { len = Mmsg(msg, _("\nDevice \"%s\" is not open or does not exist.\n"), name); @@ -165,7 +320,6 @@ static void list_one_device(char *name, DEVICE *dev, STATUS_PKT *sp) dev->print_type(), dev->print_name()); sendit(msg, len, sp); } - send_blocked_status(dev, sp); if (dev->can_append()) { bpb = dev->VolCatInfo.VolCatBlocks; if (bpb <= 0) { @@ -197,13 +351,12 @@ static void list_one_device(char *name, DEVICE *dev, STATUS_PKT *sp) edit_uint64_with_commas(dev->file, b1), edit_uint64_with_commas(dev->block_num, b2)); sendit(msg, len, sp); - } else { len = Mmsg(msg, _("\nDevice %s: %s is not open.\n"), dev->print_type(), dev->print_name()); sendit(msg, len, sp); - send_blocked_status(dev, sp); } + send_blocked_status(dev, sp); /* TODO: We need to check with Mount command, maybe we can * display this number only when the device is open. @@ -213,12 +366,15 @@ static void list_one_device(char *name, DEVICE *dev, STATUS_PKT *sp) uint64_t f, t; dev->get_freespace(&f, &t); if (t > 0) { /* We might not have access to numbers */ - len = Mmsg(msg, _(" Available Space=%sB\n"), + len = Mmsg(msg, _(" Available %sSpace=%sB\n"), + dev->is_cloud() ? _("Cache ") : "", edit_uint64_with_suffix(f, ed1)); sendit(msg, len, sp); } } + dev->show_tape_alerts((DCR *)sp, list_short, list_all, status_alert_callback); + if (!sp->api) sendit("==\n", 4, sp); } @@ -236,20 +392,45 @@ void _dbg_list_one_device(char *name, DEVICE *dev, const char *file, int line) static void list_one_autochanger(char *name, AUTOCHANGER *changer, STATUS_PKT *sp) { int len; + char *p; DEVRES *device; POOL_MEM msg(PM_MESSAGE); + OutputWriter ow(sp->api_opts); - len = Mmsg(msg, _("Autochanger \"%s\" with devices:\n"), - changer->hdr.name); - sendit(msg, len, sp); + if (sp->api > 1) { + ow.get_output(OT_START_OBJ, + OT_STRING, "autochanger", changer->hdr.name, + OT_END); - foreach_alist(device, changer->device) { - if (device->dev) { - len = Mmsg(msg, " %s\n", device->dev->print_name()); - sendit(msg, len, sp); - } else { - len = Mmsg(msg, " %s\n", device->hdr.name); - sendit(msg, len, sp); + ow.start_group("devices"); + + foreach_alist(device, changer->device) { + ow.get_output(OT_START_OBJ, + OT_STRING, "name", device->hdr.name, + OT_STRING, "device",device->device_name, + OT_END_OBJ, + OT_END); + } + + ow.end_group(); + + p = ow.get_output(OT_END_OBJ, OT_END); + sendit(p, strlen(p), sp); + + } else { + + len = Mmsg(msg, _("Autochanger \"%s\" with devices:\n"), + changer->hdr.name); + sendit(msg, len, sp); + + foreach_alist(device, changer->device) { + if (device->dev) { + len = Mmsg(msg, " %s\n", device->dev->print_name()); + sendit(msg, len, sp); + } else { + len = Mmsg(msg, " %s\n", device->hdr.name); + sendit(msg, len, sp); + } } } } @@ -280,6 +461,60 @@ static void list_devices(STATUS_PKT *sp, char *name) if (!sp->api) sendit("====\n\n", 6, sp); } +static void list_cloud_transfers(STATUS_PKT *sp, bool verbose) +{ + bool first=true; + int len; + DEVRES *device; + POOL_MEM msg(PM_MESSAGE); + + foreach_res(device, R_DEVICE) { + if (device->dev && device->dev->is_cloud()) { + + if (first) { + if (!sp->api) { + len = Mmsg(msg, _("Cloud transfer status:\n")); + sendit(msg, len, sp); + } + first = false; + } + + cloud_dev *cdev = (cloud_dev*)device->dev; + len = cdev->get_cloud_upload_transfer_status(msg, verbose); + sendit(msg, len, sp); + len = cdev->get_cloud_download_transfer_status(msg, verbose); + sendit(msg, len, sp); + break; /* only once, transfer mgr are shared */ + } + } + + if (!first && !sp->api) sendit("====\n\n", 6, sp); +} + +static void api_list_sd_status_header(STATUS_PKT *sp) +{ + char *p; + alist drivers(10, not_owned_by_alist); + OutputWriter wt(sp->api_opts); + + sd_list_loaded_drivers(&drivers); + wt.start_group("header"); + wt.get_output( + OT_STRING, "name", my_name, + OT_STRING, "version", VERSION " (" BDATE ")", + OT_STRING, "uname", HOST_OS " " DISTNAME " " DISTVER, + OT_UTIME, "started", daemon_start_time, + OT_INT, "jobs_run", num_jobs_run, + OT_INT, "jobs_running",job_count(), + OT_INT, "ndevices", ((rblist *)res_head[R_DEVICE-r_first]->res_list)->size(), + OT_INT, "nautochgr", ((rblist *)res_head[R_AUTOCHANGER-r_first]->res_list)->size(), + OT_PLUGINS,"plugins", b_plugin_list, + OT_ALIST_STR, "drivers", &drivers, + OT_END); + p = wt.end_group(); + sendit(p, strlen(p), sp); +} + static void list_status_header(STATUS_PKT *sp) { char dt[MAX_TIME_LENGTH]; @@ -287,6 +522,11 @@ static void list_status_header(STATUS_PKT *sp) POOL_MEM msg(PM_MESSAGE); int len; + if (sp->api) { + api_list_sd_status_header(sp); + return; + } + len = Mmsg(msg, _("%s %sVersion: %s (%s) %s %s %s\n"), my_name, "", VERSION, BDATE, HOST_OS, DISTNAME, DISTVER); sendit(msg, len, sp); @@ -305,9 +545,13 @@ static void list_status_header(STATUS_PKT *sp) edit_uint64_with_commas(sm_max_buffers, b5)); sendit(msg, len, sp); len = Mmsg(msg, " Sizes: boffset_t=%d size_t=%d int32_t=%d int64_t=%d " - "mode=%d,%d\n", + "mode=%d,%d newbsr=%d\n", (int)sizeof(boffset_t), (int)sizeof(size_t), (int)sizeof(int32_t), - (int)sizeof(int64_t), (int)DEVELOPER_MODE, (int)0); + (int)sizeof(int64_t), (int)DEVELOPER_MODE, 0, use_new_match_all); + sendit(msg, len, sp); + len = Mmsg(msg, _(" Res: ndevices=%d nautochgr=%d\n"), + ((rblist *)res_head[R_DEVICE-r_first]->res_list)->size(), + ((rblist *)res_head[R_AUTOCHANGER-r_first]->res_list)->size()); sendit(msg, len, sp); list_plugins(sp); } @@ -323,16 +567,16 @@ static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp) return; } if (!dev->enabled) { - len = Mmsg(msg, _(" Device is disabled. User command.\n")); + len = Mmsg(msg, _(" Device is disabled. User command.\n")); sendit(msg, len, sp); } switch (dev->blocked()) { case BST_UNMOUNTED: - len = Mmsg(msg, _(" Device is BLOCKED. User unmounted.\n")); + len = Mmsg(msg, _(" Device is BLOCKED. User unmounted.\n")); sendit(msg, len, sp); break; case BST_UNMOUNTED_WAITING_FOR_SYSOP: - len = Mmsg(msg, _(" Device is BLOCKED. User unmounted during wait for media/mount.\n")); + len = Mmsg(msg, _(" Device is BLOCKED. User unmounted during wait for media/mount.\n")); sendit(msg, len, sp); break; case BST_WAITING_FOR_SYSOP: @@ -343,7 +587,7 @@ static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp) dev->Lock_dcrs(); foreach_dlist(dcr, dev->attached_dcrs) { if (dcr->jcr->JobStatus == JS_WaitMount) { - len = Mmsg(msg, _(" Device is BLOCKED waiting for mount of volume \"%s\",\n" + len = Mmsg(msg, _(" Device is BLOCKED waiting for mount of volume \"%s\",\n" " Pool: %s\n" " Media type: %s\n"), dcr->VolumeName, @@ -352,7 +596,7 @@ static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp) sendit(msg, len, sp); found_jcr = true; } else if (dcr->jcr->JobStatus == JS_WaitMedia) { - len = Mmsg(msg, _(" Device is BLOCKED waiting to create a volume for:\n" + len = Mmsg(msg, _(" Device is BLOCKED waiting to create a volume for:\n" " Pool: %s\n" " Media type: %s\n"), dcr->pool_name, @@ -364,17 +608,17 @@ static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp) dev->Unlock_dcrs(); dev->Unlock(); if (!found_jcr) { - len = Mmsg(msg, _(" Device is BLOCKED waiting for media.\n")); + len = Mmsg(msg, _(" Device is BLOCKED waiting for media.\n")); sendit(msg, len, sp); } } break; case BST_DOING_ACQUIRE: - len = Mmsg(msg, _(" Device is being initialized.\n")); + len = Mmsg(msg, _(" Device is being initialized.\n")); sendit(msg, len, sp); break; case BST_WRITING_LABEL: - len = Mmsg(msg, _(" Device is blocked labeling a Volume.\n")); + len = Mmsg(msg, _(" Device is blocked labeling a Volume.\n")); sendit(msg, len, sp); break; default: @@ -383,11 +627,11 @@ static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp) /* Send autochanger slot status */ if (dev->is_autochanger()) { if (dev->get_slot() > 0) { - len = Mmsg(msg, _(" Slot %d %s loaded in drive %d.\n"), + len = Mmsg(msg, _(" Slot %d %s loaded in drive %d.\n"), dev->get_slot(), dev->is_open()?"is": "was last", dev->drive_index); sendit(msg, len, sp); } else if (dev->get_slot() <= 0) { - len = Mmsg(msg, _(" Drive %d is not loaded.\n"), dev->drive_index); + len = Mmsg(msg, _(" Drive %d is not loaded.\n"), dev->drive_index); sendit(msg, len, sp); } } @@ -402,11 +646,13 @@ void send_device_status(DEVICE *dev, STATUS_PKT *sp) int len; DCR *dcr = NULL; bool found = false; + char b1[35]; + if (chk_dbglvl(5)) { len = Mmsg(msg, _("Configured device capabilities:\n")); sendit(msg, len, sp); - len = Mmsg(msg, " %sEOF %sBSR %sBSF %sFSR %sFSF %sEOM %sREM %sRACCESS %sAUTOMOUNT %sLABEL %sANONVOLS %sALWAYSOPEN\n", + len = Mmsg(msg, " %sEOF %sBSR %sBSF %sFSR %sFSF %sEOM %sREM %sRACCESS %sAUTOMOUNT %sLABEL %sANONVOLS %sALWAYSOPEN\n", dev->capabilities & CAP_EOF ? "" : "!", dev->capabilities & CAP_BSR ? "" : "!", dev->capabilities & CAP_BSF ? "" : "!", @@ -424,7 +670,7 @@ void send_device_status(DEVICE *dev, STATUS_PKT *sp) len = Mmsg(msg, _("Device state:\n")); sendit(msg, len, sp); - len = Mmsg(msg, " %sOPENED %sTAPE %sLABEL %sMALLOC %sAPPEND %sREAD %sEOT %sWEOT %sEOF %sNEXTVOL %sSHORT %sMOUNTED\n", + len = Mmsg(msg, " %sOPENED %sTAPE %sLABEL %sMALLOC %sAPPEND %sREAD %sEOT %sWEOT %sEOF %sNEXTVOL %sSHORT %sMOUNTED\n", dev->is_open() ? "" : "!", dev->is_tape() ? "" : "!", dev->is_labeled() ? "" : "!", @@ -438,8 +684,10 @@ void send_device_status(DEVICE *dev, STATUS_PKT *sp) dev->state & ST_SHORT ? "" : "!", dev->state & ST_MOUNTED ? "" : "!"); sendit(msg, len, sp); - len = Mmsg(msg, _(" num_writers=%d reserves=%d block=%d enabled=%d\n"), dev->num_writers, - dev->num_reserved(), dev->blocked(), dev->enabled); + len = Mmsg(msg, _(" Writers=%d reserves=%d blocked=%d enabled=%d usage=%s\n"), dev->num_writers, + dev->num_reserved(), dev->blocked(), dev->enabled, + edit_uint64_with_commas(dev->usage, b1)); + sendit(msg, len, sp); len = Mmsg(msg, _("Attached JobIds: ")); @@ -462,15 +710,121 @@ void send_device_status(DEVICE *dev, STATUS_PKT *sp) len = Mmsg(msg, _("Device parameters:\n")); sendit(msg, len, sp); - len = Mmsg(msg, _(" Archive name: %s Device name: %s\n"), dev->archive_name(), + len = Mmsg(msg, _(" Archive name: %s Device name: %s\n"), dev->archive_name(), dev->name()); sendit(msg, len, sp); - len = Mmsg(msg, _(" File=%u block=%u\n"), dev->file, dev->block_num); + len = Mmsg(msg, _(" File=%u block=%u\n"), dev->file, dev->block_num); sendit(msg, len, sp); - len = Mmsg(msg, _(" Min block=%u Max block=%u\n"), dev->min_block_size, dev->max_block_size); + len = Mmsg(msg, _(" Min block=%u Max block=%u\n"), dev->min_block_size, dev->max_block_size); sendit(msg, len, sp); } +static void api_list_running_jobs(STATUS_PKT *sp) +{ + char *p1, *p2, *p3; + int i1, i2, i3; + OutputWriter ow(sp->api_opts); + + uint64_t inst_bps, total_bps; + int inst_sec, total_sec; + JCR *jcr; + DCR *dcr, *rdcr; + time_t now = time(NULL); + + foreach_jcr(jcr) { + if (jcr->getJobType() == JT_SYSTEM) { + continue; + } + ow.get_output(OT_CLEAR, + OT_START_OBJ, + OT_INT32, "jobid", jcr->JobId, + OT_STRING, "job", jcr->Job, + OT_JOBLEVEL,"level", jcr->getJobLevel(), + OT_JOBTYPE, "type", jcr->getJobType(), + OT_JOBSTATUS,"status", jcr->JobStatus, + OT_PINT64, "jobbytes", jcr->JobBytes, + OT_INT32, "jobfiles", jcr->JobFiles, + OT_UTIME, "starttime", jcr->start_time, + OT_INT32, "errors", jcr->JobErrors, + OT_INT32, "newbsr", (int32_t)jcr->use_new_match_all, + OT_END); + + dcr = jcr->dcr; + rdcr = jcr->read_dcr; + + p1 = p2 = p3 = NULL; + if (rdcr && rdcr->device) { + p1 = rdcr->VolumeName; + p2 = rdcr->pool_name; + p3 = rdcr->device->hdr.name; + } + ow.get_output(OT_STRING, "read_volume", NPRTB(p1), + OT_STRING, "read_pool", NPRTB(p2), + OT_STRING, "read_device", NPRTB(p3), + OT_END); + + p1 = p2 = p3 = NULL; + i1 = i2 = i3 = 0; + if (dcr && dcr->device) { + p1 = dcr->VolumeName; + p2 = dcr->pool_name; + p3 = dcr->device->hdr.name; + i1 = dcr->spooling; + i2 = dcr->despooling; + i3 = dcr->despool_wait; + } + + ow.get_output(OT_STRING, "write_volume", NPRTB(p1), + OT_STRING, "write_pool", NPRTB(p2), + OT_STRING, "write_device", NPRTB(p3), + OT_INT, "spooling", i1, + OT_INT, "despooling", i2, + OT_INT, "despool_wait", i3, + OT_END); + + if (jcr->last_time == 0) { + jcr->last_time = jcr->run_time; + } + + total_sec = now - jcr->run_time; + inst_sec = now - jcr->last_time; + + if (total_sec <= 0) { + total_sec = 1; + } + if (inst_sec <= 0) { + inst_sec = 1; + } + + /* Instanteous bps not smoothed */ + inst_bps = (jcr->JobBytes - jcr->LastJobBytes) / inst_sec; + if (jcr->LastRate == 0) { + jcr->LastRate = inst_bps; + } + + /* Smooth the instantaneous bps a bit */ + inst_bps = (2 * jcr->LastRate + inst_bps) / 3; + /* total bps (AveBytes/sec) since start of job */ + total_bps = jcr->JobBytes / total_sec; + + p1 = ow.get_output(OT_PINT64, "avebytes_sec", total_bps, + OT_PINT64, "lastbytes_sec", inst_bps, + OT_END_OBJ, + OT_END); + + sendit(p1, strlen(p1), sp); + + /* Update only every 10 seconds */ + if (now - jcr->last_time > 10) { + jcr->LastRate = inst_bps; + jcr->LastJobBytes = jcr->JobBytes; + jcr->last_time = now; + } + } + endeach_jcr(jcr); + +} + static void list_running_jobs(STATUS_PKT *sp) { bool found = false; @@ -484,6 +838,11 @@ static void list_running_jobs(STATUS_PKT *sp) POOL_MEM msg(PM_MESSAGE); time_t now = time(NULL); + if (sp->api > 1) { + api_list_running_jobs(sp); + return; + } + len = Mmsg(msg, _("\nRunning Jobs:\n")); if (!sp->api) sendit(msg, len, sp); @@ -506,7 +865,7 @@ static void list_running_jobs(STATUS_PKT *sp) } if (rdcr && rdcr->device) { len = Mmsg(msg, _("Reading: %s %s job %s JobId=%d Volume=\"%s\"\n" - " pool=\"%s\" device=%s\n"), + " pool=\"%s\" device=%s newbsr=%d\n"), job_level_to_str(jcr->getJobLevel()), job_type_to_str(jcr->getJobType()), JobName, @@ -514,7 +873,9 @@ static void list_running_jobs(STATUS_PKT *sp) rdcr->VolumeName, rdcr->pool_name, rdcr->dev?rdcr->dev->print_name(): - rdcr->device->device_name); + rdcr->device->device_name, + jcr->use_new_match_all + ); sendit(msg, len, sp); } else if (dcr && dcr->device) { len = Mmsg(msg, _("Writing: %s %s job %s JobId=%d Volume=\"%s\"\n" @@ -739,6 +1100,8 @@ bool qstatus_cmd(JCR *jcr) } else if (strcasecmp(cmd, "resources") == 0) { sp.api = api; list_resources(&sp); + } else if (strcasecmp(cmd, "cloud") == 0) { + list_cloud_transfers(&sp, true); } else { pm_strcpy(jcr->errmsg, dir->msg); dir->fsend(_("3900 Unknown arg in .status command: %s\n"), jcr->errmsg); @@ -750,12 +1113,15 @@ bool qstatus_cmd(JCR *jcr) return ret; } +/* List plugins and drivers */ static void list_plugins(STATUS_PKT *sp) { POOL_MEM msg(PM_MESSAGE); + alist drivers(10, not_owned_by_alist); + int len; + if (b_plugin_list && b_plugin_list->size() > 0) { Plugin *plugin; - int len; pm_strcpy(msg, " Plugin: "); foreach_alist(plugin, b_plugin_list) { len = pm_strcat(msg, plugin->file); @@ -774,4 +1140,19 @@ static void list_plugins(STATUS_PKT *sp) len = pm_strcat(msg, "\n"); sendit(msg.c_str(), len, sp); } + sd_list_loaded_drivers(&drivers); + if (drivers.size() > 0) { + char *drv; + pm_strcpy(msg, " Drivers: "); + foreach_alist(drv, (&drivers)) { + len = pm_strcat(msg, drv); + if (len > 80) { + pm_strcat(msg, "\n "); + } else { + pm_strcat(msg, " "); + } + } + len = pm_strcat(msg, "\n"); + sendit(msg.c_str(), len, sp); + } } diff --git a/bacula/src/stored/stored.c b/bacula/src/stored/stored.c index a8d9dfc19d..5886433d9a 100644 --- a/bacula/src/stored/stored.c +++ b/bacula/src/stored/stored.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,13 +11,13 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * Second generation Storage daemon. + * Third generation Storage daemon. * * Written by Kern Sibbald, MM * @@ -37,9 +37,13 @@ */ #include "sd_plugins.h" -/* Imported functions */ +/* Imported functions and variables */ extern bool parse_sd_config(CONFIG *config, const char *configfile, int exit_code); +extern dlist *daemon_msg_queue; +extern pthread_mutex_t daemon_msg_queue_mutex; + + /* Forward referenced functions */ void terminate_stored(int sig); static int check_resources(); @@ -52,11 +56,6 @@ extern "C" void *device_initialization(void *arg); /* Global variables exported */ char OK_msg[] = "3000 OK\n"; char TERM_msg[] = "3999 Terminate\n"; -STORES *me = NULL; /* our Global resource */ - -bool forge_on = false; /* proceed inspite of I/O errors */ -pthread_mutex_t device_release_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t wait_device_release = PTHREAD_COND_INITIALIZER; void *start_heap; static bool test_config = false; @@ -98,6 +97,24 @@ static void usage() exit(1); } +/* + * !!! WARNING !!! Use this function only when bacula is stopped. + * ie, after a fatal signal and before exiting the program + * Print information about a JCR + */ +static void sd_debug_print(JCR *jcr, FILE *fp) +{ + if (jcr->dcr) { + DCR *dcr = jcr->dcr; + fprintf(fp, "\tdcr=%p volumename=%s dev=%p newvol=%d reserved=%d locked=%d\n", + dcr, dcr->VolumeName, dcr->dev, dcr->NewVol, + dcr->is_reserved(), + dcr->is_dev_locked()); + } else { + fprintf(fp, "dcr=*None*\n"); + } +} + /********************************************************************* * * Main Bacula Unix Storage Daemon @@ -114,6 +131,7 @@ int main (int argc, char *argv[]) pthread_t thid; char *uid = NULL; char *gid = NULL; + MQUEUE_ITEM *item = NULL; start_heap = sbrk(0); setlocale(LC_ALL, ""); @@ -124,17 +142,19 @@ int main (int argc, char *argv[]) my_name_is(argc, argv, "bacula-sd"); init_msg(NULL, NULL); daemon_start_time = time(NULL); + /* Setup daemon message queue */ + daemon_msg_queue = New(dlist(item, &item->link)); /* Sanity checks */ if (TAPE_BSIZE % B_DEV_BSIZE != 0 || TAPE_BSIZE / B_DEV_BSIZE == 0) { - Emsg2(M_ABORT, 0, _("Tape block size (%d) not multiple of system size (%d)\n"), + Jmsg2(NULL, M_ABORT, 0, _("Tape block size (%d) not multiple of system size (%d)\n"), TAPE_BSIZE, B_DEV_BSIZE); } if (TAPE_BSIZE != (1 << (ffs(TAPE_BSIZE)-1))) { - Emsg1(M_ABORT, 0, _("Tape block size (%d) is not a power of 2\n"), TAPE_BSIZE); + Jmsg1(NULL, M_ABORT, 0, _("Tape block size (%d) is not a power of 2\n"), TAPE_BSIZE); } - while ((ch = getopt(argc, argv, "c:d:fg:mpstu:v?T")) != -1) { + while ((ch = getopt(argc, argv, "c:d:fg:mpstu:v?Ti")) != -1) { switch (ch) { case 'c': /* configuration file */ if (configfile != NULL) { @@ -198,6 +218,11 @@ int main (int argc, char *argv[]) verbose++; break; + /* Temp code to enable new match_bsr() code, not documented */ + case 'i': + use_new_match_all = 1; + break; + case '?': default: usage(); @@ -215,10 +240,11 @@ int main (int argc, char *argv[]) argc--; argv++; } - if (argc) + if (argc) { usage(); + } - if (!foreground) { + if (!foreground && !test_config) { daemon_start(); /* become daemon */ init_stack_dump(); /* pick up new pid */ } @@ -231,7 +257,7 @@ int main (int argc, char *argv[]) configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); + config = New(CONFIG()); parse_sd_config(config, configfile, M_ERROR_TERM); if (init_crypto() != 0) { @@ -281,11 +307,12 @@ int main (int argc, char *argv[]) create_volume_lists(); /* do before device_init */ if (pthread_create(&thid, NULL, device_initialization, NULL) != 0) { berrno be; - Emsg1(M_ABORT, 0, _("Unable to create thread. ERR=%s\n"), be.bstrerror()); + Jmsg1(NULL, M_ABORT, 0, _("Unable to create thread. ERR=%s\n"), be.bstrerror()); } start_watchdog(); /* start watchdog thread */ init_jcr_subsystem(); /* start JCR watchdogs etc. */ + dbg_jcr_add_hook(sd_debug_print); /* used to director variables */ /* Single server used for Director and File daemon */ server_tid = pthread_self(); @@ -312,6 +339,8 @@ static int check_resources() { bool OK = true; bool tls_needed; + AUTOCHANGER *changer; + DEVRES *device; me = (STORES *)GetNextRes(R_STORAGE, NULL); if (!me) { @@ -453,8 +482,15 @@ static int check_resources() } } - OK = init_autochangers(); + foreach_res(changer, R_AUTOCHANGER) { + foreach_alist(device, changer->device) { + device->cap_bits |= CAP_AUTOCHANGER; + } + } + if (OK) { + OK = init_autochangers(); + } if (OK) { close_msg(NULL); /* close temp message handler */ @@ -554,6 +590,7 @@ void *device_initialization(void *arg) DCR *dcr; JCR *jcr; DEVICE *dev; + struct stat statp; LockRes(); @@ -569,17 +606,34 @@ void *device_initialization(void *arg) } foreach_res(device, R_DEVICE) { - Dmsg1(90, "calling init_dev %s\n", device->device_name); + Dmsg1(90, "calling init_dev %s\n", device->hdr.name); dev = init_dev(NULL, device); - Dmsg1(10, "SD init done %s\n", device->device_name); + Dmsg1(10, "SD init done %s\n", device->hdr.name); if (!dev) { - Jmsg1(NULL, M_ERROR, 0, _("Could not initialize %s\n"), device->device_name); + Jmsg1(NULL, M_ERROR, 0, _("Could not initialize SD device \"%s\"\n"), device->hdr.name); continue; } jcr->dcr = dcr = new_dcr(jcr, NULL, dev); generate_plugin_event(jcr, bsdEventDeviceInit, dcr); + if (device->control_name && stat(device->control_name, &statp) < 0) { + berrno be; + Jmsg2(jcr, M_ERROR_TERM, 0, _("Unable to stat ControlDevice %s: ERR=%s\n"), + device->control_name, be.bstrerror()); + } + + if ((device->lock_command && device->control_name) && + !me->plugin_directory) { + Jmsg0(jcr, M_ERROR_TERM, 0, _("No plugin directory configured for SAN shared storage\n")); + } + + + /* + * Note: be careful setting the slot here. If the drive + * is shared storage, the contents can change before + * the drive is used. + */ if (device->cap_bits & CAP_ALWAYSOPEN) { if (dev->is_autochanger()) { /* If autochanger set slot in dev sturcture */ @@ -604,7 +658,7 @@ void *device_initialization(void *arg) dev->clear_slot(); } if (device->cap_bits & CAP_AUTOMOUNT && dev->is_open()) { - switch (read_dev_volume_label(dcr)) { + switch (dev->read_dev_volume_label(dcr)) { case VOL_OK: memcpy(&dev->VolCatInfo, &dcr->VolCatInfo, sizeof(dev->VolCatInfo)); volume_unused(dcr); /* mark volume "released" */ @@ -702,14 +756,19 @@ void terminate_stored(int sig) unload_plugins(); free_volume_lists(); + P(daemon_msg_queue_mutex); + daemon_msg_queue->destroy(); + free(daemon_msg_queue); + V(daemon_msg_queue_mutex); + foreach_res(device, R_DEVICE) { - Dmsg1(10, "Term device %s\n", device->device_name); + Dmsg2(10, "Term device %s %s\n", device->hdr.name, device->device_name); if (device->dev) { device->dev->clear_volhdr(); - device->dev->term(); + device->dev->term(NULL); device->dev = NULL; } else { - Dmsg1(10, "No dev structure %s\n", device->device_name); + Dmsg2(10, "No dev structure %s %s\n", device->hdr.name, device->device_name); } } if (server_tid_valid) { @@ -721,8 +780,7 @@ void terminate_stored(int sig) configfile = NULL; } if (config) { - config->free_resources(); - free(config); + delete config; config = NULL; } @@ -732,6 +790,8 @@ void terminate_stored(int sig) term_msg(); cleanup_crypto(); term_reservations_lock(); + free(res_head); + res_head = NULL; close_memory_pool(); lmgr_cleanup_main(); diff --git a/bacula/src/stored/stored.h b/bacula/src/stored/stored.h index bc03b25bf6..682ce58b0a 100644 --- a/bacula/src/stored/stored.h +++ b/bacula/src/stored/stored.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -41,6 +41,8 @@ const int sd_dbglvl = 300; const int sd_dbglvl = 300; #endif +#undef SD_DEDUP_SUPPORT + #ifdef HAVE_MTIO_H #include #else @@ -59,8 +61,6 @@ const int sd_dbglvl = 300; #include "block.h" #include "record.h" #include "dev.h" -#include "file_dev.h" -#include "tape_dev.h" #include "stored_conf.h" #include "bsr.h" #include "jcr.h" @@ -87,7 +87,11 @@ int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); #include "file_dev.h" #include "tape_dev.h" +#include "fifo_dev.h" +#include "null_dev.h" #include "vtape_dev.h" +#include "cloud_dev.h" +#include "aligned_dev.h" #include "sd_plugins.h" /* Daemon globals from stored.c */ diff --git a/bacula/src/stored/stored_conf.c b/bacula/src/stored/stored_conf.c index 99da6309ab..43a47d9a2a 100644 --- a/bacula/src/stored/stored_conf.c +++ b/bacula/src/stored/stored_conf.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -24,12 +24,12 @@ #include "bacula.h" #include "stored.h" +#include "cloud_driver.h" /* First and last resource ids */ int32_t r_first = R_FIRST; int32_t r_last = R_LAST; -static RES *sres_head[R_LAST - R_FIRST + 1]; -RES **res_head = sres_head; +RES_HEAD **res_head; /* We build the current resource here statically, * then move it to dynamic memory */ @@ -78,6 +78,7 @@ static RES_ITEM store_items[] = { {"TlsAllowedCn", store_alist_str, ITEM(res_store.tls_allowed_cns), 0, 0, 0}, {"ClientConnectWait", store_time, ITEM(res_store.client_wait), 0, ITEM_DEFAULT, 30 * 60}, {"VerId", store_str, ITEM(res_store.verid), 0, 0, 0}, + {"CommCompression", store_bool, ITEM(res_store.comm_compression), 0, ITEM_DEFAULT, true}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -108,6 +109,7 @@ static RES_ITEM dev_items[] = { {"MediaType", store_strname,ITEM(res_dev.media_type), 0, ITEM_REQUIRED, 0}, {"DeviceType", store_devtype,ITEM(res_dev.dev_type), 0, 0, 0}, {"ArchiveDevice", store_strname,ITEM(res_dev.device_name), 0, ITEM_REQUIRED, 0}, + {"AlignedDevice", store_strname,ITEM(res_dev.adevice_name), 0, 0, 0}, {"HardwareEndOfFile", store_bit, ITEM(res_dev.cap_bits), CAP_EOF, ITEM_DEFAULT, 1}, {"HardwareEndOfMedium", store_bit, ITEM(res_dev.cap_bits), CAP_EOM, ITEM_DEFAULT, 1}, {"BackwardSpaceRecord", store_bit, ITEM(res_dev.cap_bits), CAP_BSR, ITEM_DEFAULT, 1}, @@ -137,6 +139,7 @@ static RES_ITEM dev_items[] = { {"ControlDevice", store_strname,ITEM(res_dev.control_name), 0, 0, 0}, {"ChangerCommand", store_strname,ITEM(res_dev.changer_command), 0, 0, 0}, {"AlertCommand", store_strname,ITEM(res_dev.alert_command), 0, 0, 0}, + {"LockCommand", store_strname,ITEM(res_dev.lock_command), 0, 0, 0}, {"MaximumChangerWait", store_time, ITEM(res_dev.max_changer_wait), 0, ITEM_DEFAULT, 5 * 60}, {"MaximumOpenWait", store_time, ITEM(res_dev.max_open_wait), 0, ITEM_DEFAULT, 5 * 60}, {"MaximumNetworkBufferSize", store_pint32, ITEM(res_dev.max_network_buffer_size), 0, 0, 0}, @@ -146,6 +149,7 @@ static RES_ITEM dev_items[] = { {"MaximumBlockSize", store_maxblocksize, ITEM(res_dev.max_block_size), 0, 0, 0}, {"PaddingSize", store_size32, ITEM(res_dev.padding_size), 0, ITEM_DEFAULT, 4096}, {"FileAlignment", store_size32, ITEM(res_dev.file_alignment), 0, ITEM_DEFAULT, 4096}, + {"MinimumAlignedSize", store_size32, ITEM(res_dev.min_aligned_size), 0, ITEM_DEFAULT, 4096}, {"MaximumVolumeSize", store_size64, ITEM(res_dev.max_volume_size), 0, 0, 0}, {"MaximumFileSize", store_size64, ITEM(res_dev.max_file_size), 0, ITEM_DEFAULT, 1000000000}, {"VolumeCapacity", store_size64, ITEM(res_dev.volume_capacity), 0, 0, 0}, @@ -162,6 +166,7 @@ static RES_ITEM dev_items[] = { {"WritePartCommand", store_strname,ITEM(res_dev.write_part_command), 0, 0, 0}, {"FreeSpaceCommand", store_strname,ITEM(res_dev.free_space_command), 0, 0, 0}, {"LabelType", store_label, ITEM(res_dev.label_type), 0, 0, 0}, + {"Cloud", store_res, ITEM(res_dev.cloud), R_CLOUD, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -172,11 +177,30 @@ static RES_ITEM changer_items[] = { {"Device", store_alist_res, ITEM(res_changer.device), R_DEVICE, ITEM_REQUIRED, 0}, {"ChangerDevice", store_strname, ITEM(res_changer.changer_name), 0, ITEM_REQUIRED, 0}, {"ChangerCommand", store_strname, ITEM(res_changer.changer_command), 0, ITEM_REQUIRED, 0}, + {"LockCommand", store_strname,ITEM(res_changer.lock_command), 0, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; - -// {"mountanonymousvolumes", store_bit, ITEM(res_dev.cap_bits), CAP_ANONVOLS, ITEM_DEFAULT, 0}, +/* Cloud driver definition */ +static RES_ITEM cloud_items[] = { + {"Name", store_name, ITEM(res_cloud.hdr.name), 0, ITEM_REQUIRED, 0}, + {"Description", store_str, ITEM(res_cloud.hdr.desc), 0, 0, 0}, + {"Driver", store_cloud_driver, ITEM(res_cloud.driver_type), 0, ITEM_REQUIRED, 0}, + {"HostName", store_strname,ITEM(res_cloud.host_name), 0, ITEM_REQUIRED, 0}, + {"BucketName", store_strname,ITEM(res_cloud.bucket_name), 0, ITEM_REQUIRED, 0}, + {"Region", store_strname,ITEM(res_cloud.region), 0, 0, 0}, + {"AccessKey", store_strname,ITEM(res_cloud.access_key), 0, ITEM_REQUIRED, 0}, + {"SecretKey", store_strname,ITEM(res_cloud.secret_key), 0, ITEM_REQUIRED, 0}, + {"Protocol", store_protocol, ITEM(res_cloud.protocol), 0, ITEM_DEFAULT, 0}, /* HTTPS */ + {"UriStyle", store_uri_style, ITEM(res_cloud.uri_style), 0, ITEM_DEFAULT, 0}, /* VirtualHost */ + {"TruncateCache", store_truncate, ITEM(res_cloud.trunc_opt), 0, ITEM_DEFAULT, TRUNC_NO}, + {"Upload", store_upload, ITEM(res_cloud.upload_opt), 0, ITEM_DEFAULT, UPLOAD_NO}, + {"MaximumConcurrentUploads", store_pint32, ITEM(res_cloud.max_concurrent_uploads), 0, ITEM_DEFAULT, 0}, + {"MaximumConcurrentDownloads", store_pint32, ITEM(res_cloud.max_concurrent_downloads), 0, ITEM_DEFAULT, 0}, + {"MaximumUploadBandwidth", store_speed, ITEM(res_cloud.upload_limit), 0, 0, 0}, + {"MaximumDownloadBandwidth", store_speed, ITEM(res_cloud.download_limit), 0, 0, 0}, + {NULL, NULL, {0}, 0, 0, 0} +}; /* Message resource */ @@ -190,6 +214,7 @@ RES_TABLE resources[] = { {"Device", dev_items, R_DEVICE}, {"Messages", msgs_items, R_MSGS}, {"Autochanger", changer_items, R_AUTOCHANGER}, + {"Cloud", cloud_items, R_CLOUD}, {NULL, NULL, 0} }; @@ -201,38 +226,218 @@ RES_TABLE resources[] = { s_kw dev_types[] = { {"File", B_FILE_DEV}, {"Tape", B_TAPE_DEV}, - {"Dvd", B_DVD_DEV}, {"Fifo", B_FIFO_DEV}, - {"Vtl", B_VTL_DEV}, {"VTape", B_VTAPE_DEV}, + {"Vtl", B_VTL_DEV}, + {"Aligned", B_ALIGNED_DEV}, + {"Null", B_NULL_DEV}, + {"Cloud", B_CLOUD_DEV}, {NULL, 0} }; /* - * Store Device Type (File, FIFO, Tape, DVD) + * Store Device Type (File, FIFO, Tape, Cloud, ...) * */ void store_devtype(LEX *lc, RES_ITEM *item, int index, int pass) { - int i; + bool found = false; lex_get_token(lc, T_NAME); /* Store the label pass 2 so that type is defined */ - for (i=0; dev_types[i].name; i++) { + for (int i=0; dev_types[i].name; i++) { if (strcasecmp(lc->str, dev_types[i].name) == 0) { *(uint32_t *)(item->value) = dev_types[i].token; - i = 0; + found = true; break; } } - if (i != 0) { + if (!found) { scan_err1(lc, _("Expected a Device Type keyword, got: %s"), lc->str); } scan_to_eol(lc); set_bit(index, res_all.hdr.item_present); } +/* + * Cloud drivers + * + * driver driver code + */ +s_kw cloud_drivers[] = { + {"S3", C_S3_DRIVER}, + {"File", C_FILE_DRIVER}, + {NULL, 0} +}; + +/* + * Store Device Type (File, FIFO, Tape, Cloud, ...) + * + */ +void store_cloud_driver(LEX *lc, RES_ITEM *item, int index, int pass) +{ + bool found = false; + + lex_get_token(lc, T_NAME); + /* Store the label pass 2 so that type is defined */ + for (int i=0; cloud_drivers[i].name; i++) { + if (strcasecmp(lc->str, cloud_drivers[i].name) == 0) { + *(uint32_t *)(item->value) = cloud_drivers[i].token; + found = true; + break; + } + } + if (!found) { + scan_err1(lc, _("Expected a Cloud driver keyword, got: %s"), lc->str); + } + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); +} + +/* + * Cloud Truncate cache options + * + * Option option code = token + */ +s_kw trunc_opts[] = { + {"No", TRUNC_NO}, + {"AfterUpload", TRUNC_AFTER_UPLOAD}, + {"AtEndOfJob", TRUNC_AT_ENDOFJOB}, + {NULL, 0} +}; + +/* + * Store Cloud Truncate cache option (AfterUpload, AtEndOfJob, No) + * + */ +void store_truncate(LEX *lc, RES_ITEM *item, int index, int pass) +{ + bool found = false; + + lex_get_token(lc, T_NAME); + /* Store the label pass 2 so that type is defined */ + for (int i=0; trunc_opts[i].name; i++) { + if (strcasecmp(lc->str, trunc_opts[i].name) == 0) { + *(uint32_t *)(item->value) = trunc_opts[i].token; + found = true; + break; + } + } + if (!found) { + scan_err1(lc, _("Expected a Truncate Cache option keyword, got: %s"), lc->str); + } + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); +} + +/* + * Cloud Upload options + * + * Option option code = token + */ +s_kw upload_opts[] = { + {"No", UPLOAD_NO}, + {"EachPart", UPLOAD_EACHPART}, + {"AtEndOfJob", UPLOAD_AT_ENDOFJOB}, + {NULL, 0} +}; + +/* + * Store Cloud Upload option (EachPart, AtEndOfJob, No) + * + */ +void store_upload(LEX *lc, RES_ITEM *item, int index, int pass) +{ + bool found = false; + + lex_get_token(lc, T_NAME); + /* Store the label pass 2 so that type is defined */ + for (int i=0; upload_opts[i].name; i++) { + if (strcasecmp(lc->str, upload_opts[i].name) == 0) { + *(uint32_t *)(item->value) = upload_opts[i].token; + found = true; + break; + } + } + if (!found) { + scan_err1(lc, _("Expected a Cloud Upload option keyword, got: %s"), lc->str); + } + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); +} + +/* + * Cloud connection protocol options + * + * Option option code = token + */ +s_kw proto_opts[] = { + {"HTTPS", 0}, + {"HTTP", 1}, + {NULL, 0} +}; + +/* + * Store Cloud connect protocol option (HTTPS, HTTP) + * + */ +void store_protocol(LEX *lc, RES_ITEM *item, int index, int pass) +{ + bool found = false; + + lex_get_token(lc, T_NAME); + /* Store the label pass 2 so that type is defined */ + for (int i=0; proto_opts[i].name; i++) { + if (strcasecmp(lc->str, proto_opts[i].name) == 0) { + *(uint32_t *)(item->value) = proto_opts[i].token; + found = true; + break; + } + } + if (!found) { + scan_err1(lc, _("Expected a Cloud communications protocol option keyword, got: %s"), lc->str); + } + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); +} + +/* + * Cloud Uri Style options + * + * Option option code = token + */ +s_kw uri_opts[] = { + {"VirtualHost", 0}, + {"Path", 1}, + {NULL, 0} +}; + +/* + * Store Cloud Uri Style option + * + */ +void store_uri_style(LEX *lc, RES_ITEM *item, int index, int pass) +{ + bool found = false; + + lex_get_token(lc, T_NAME); + /* Store the label pass 2 so that type is defined */ + for (int i=0; uri_opts[i].name; i++) { + if (strcasecmp(lc->str, uri_opts[i].name) == 0) { + *(uint32_t *)(item->value) = uri_opts[i].token; + found = true; + break; + } + } + if (!found) { + scan_err1(lc, _("Expected a Cloud Uri Style option keyword, got: %s"), lc->str); + } + scan_to_eol(lc); + set_bit(index, res_all.hdr.item_present); +} + + /* * Store Maximum Block Size, and check it is not greater than MAX_BLOCK_LENGTH * @@ -354,7 +559,24 @@ void dump_resource(int type, RES *rres, void sendit(void *sock, const char *fmt, bstrncat(buf, "CAP_OFFLINEUNMOUNT ", sizeof(buf)); } bstrncat(buf, "\n", sizeof(buf)); - sendit(sock, buf); + sendit(sock, buf); /* Send caps string */ + if (res->res_dev.cloud) { + sendit(sock, " --->Cloud: name=%s\n", res->res_dev.cloud->hdr.name); + } + break; + case R_CLOUD: + sendit(sock, "Cloud: name=%s Driver=%d\n" + " HostName=%s\n" + " BucketName=%s\n" + " AccessKey=%s SecretKey=%s\n" + " AuthRegion=%s\n" + " Protocol=%d UriStyle=%d\n", + res->res_cloud.hdr.name, res->res_cloud.driver_type, + res->res_cloud.host_name, + res->res_cloud.bucket_name, + res->res_cloud.access_key, res->res_cloud.secret_key, + res->res_cloud.region, + res->res_cloud.protocol, res->res_cloud.uri_style); break; case R_AUTOCHANGER: DEVRES *dev; @@ -364,8 +586,6 @@ void dump_resource(int type, RES *rres, void sendit(void *sock, const char *fmt, foreach_alist(dev, res->res_changer.device) { sendit(sock, " --->Device: name=%s\n", dev->hdr.name); } - bstrncat(buf, "\n", sizeof(buf)); - sendit(sock, buf); break; case R_MSGS: sendit(sock, "Messages: name=%s\n", res->res_msgs.hdr.name); @@ -378,9 +598,9 @@ void dump_resource(int type, RES *rres, void sendit(void *sock, const char *fmt, sendit(sock, _("Warning: unknown resource type %d\n"), type); break; } - if (recurse && res->res_dir.hdr.next) { - dump_resource(type, (RES *)res->res_dir.hdr.next, sendit, sock); - } + rres = GetNextRes(type, rres); + if (recurse && rres) + dump_resource(type, rres, sendit, sock); } /* @@ -392,14 +612,12 @@ void dump_resource(int type, RES *rres, void sendit(void *sock, const char *fmt, */ void free_resource(RES *sres, int type) { - RES *nres; URES *res = (URES *)sres; if (res == NULL) return; /* common stuff -- free the resource name */ - nres = (RES *)res->res_dir.hdr.next; if (res->res_dir.hdr.name) { free(res->res_dir.hdr.name); } @@ -445,6 +663,9 @@ void free_resource(RES *sres, int type) if (res->res_changer.changer_command) { free(res->res_changer.changer_command); } + if (res->res_changer.lock_command) { + free(res->res_changer.lock_command); + } if (res->res_changer.device) { delete res->res_changer.device; } @@ -497,6 +718,23 @@ void free_resource(RES *sres, int type) free(res->res_store.verid); } break; + case R_CLOUD: + if (res->res_cloud.host_name) { + free(res->res_cloud.host_name); + } + if (res->res_cloud.bucket_name) { + free(res->res_cloud.bucket_name); + } + if (res->res_cloud.access_key) { + free(res->res_cloud.access_key); + } + if (res->res_cloud.secret_key) { + free(res->res_cloud.secret_key); + } + if (res->res_cloud.region) { + free(res->res_cloud.region); + } + break; case R_DEVICE: if (res->res_dev.media_type) { free(res->res_dev.media_type); @@ -504,6 +742,9 @@ void free_resource(RES *sres, int type) if (res->res_dev.device_name) { free(res->res_dev.device_name); } + if (res->res_dev.adevice_name) { + free(res->res_dev.adevice_name); + } if (res->res_dev.control_name) { free(res->res_dev.control_name); } @@ -516,6 +757,9 @@ void free_resource(RES *sres, int type) if (res->res_dev.alert_command) { free(res->res_dev.alert_command); } + if (res->res_dev.lock_command) { + free(res->res_dev.lock_command); + } if (res->res_dev.spool_directory) { free(res->res_dev.spool_directory); } @@ -553,16 +797,13 @@ void free_resource(RES *sres, int type) if (res) { free(res); } - if (nres) { - free_resource(nres, type); - } } /* Save the new resource by chaining it into the head list for * the resource. If this is pass 2, we update any resource * or alist pointers. */ -void save_resource(int type, RES_ITEM *items, int pass) +bool save_resource(CONFIG *config, int type, RES_ITEM *items, int pass) { URES *res; int rindex = type - r_first; @@ -575,13 +816,15 @@ void save_resource(int type, RES_ITEM *items, int pass) for (i=0; items[i].name; i++) { if (items[i].flags & ITEM_REQUIRED) { if (!bit_is_set(i, res_all.res_dir.hdr.item_present)) { - Emsg2(M_ERROR_TERM, 0, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), - items[i].name, resources[rindex].name); - } + Mmsg(config->m_errmsg, _("\"%s\" directive is required in \"%s\" resource, but not found.\n"), + items[i].name, resources[rindex].name); + return false; + } } /* If this triggers, take a look at lib/parse_conf.h */ if (i >= MAX_RES_ITEMS) { - Emsg1(M_ERROR_TERM, 0, _("Too many directives in \"%s\" resource\n"), resources[rindex].name); + Mmsg(config->m_errmsg, _("Too many directives in \"%s\" resource\n"), resources[rindex].name); + return false; } } @@ -595,28 +838,31 @@ void save_resource(int type, RES_ITEM *items, int pass) int errstat; switch (type) { /* Resources not containing a resource */ - case R_DEVICE: case R_MSGS: + case R_CLOUD: break; /* Resources containing a resource or an alist */ case R_DIRECTOR: if ((res = (URES *)GetResWithName(R_DIRECTOR, res_all.res_dir.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Director resource %s\n"), res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Director resource %s\n"), res_all.res_dir.hdr.name); + return false; } res->res_dir.tls_allowed_cns = res_all.res_dir.tls_allowed_cns; break; case R_STORAGE: if ((res = (URES *)GetResWithName(R_STORAGE, res_all.res_dir.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find Storage resource %s\n"), res_all.res_dir.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find Storage resource %s\n"), res_all.res_dir.hdr.name); + return false; } res->res_store.messages = res_all.res_store.messages; res->res_store.tls_allowed_cns = res_all.res_store.tls_allowed_cns; break; case R_AUTOCHANGER: if ((res = (URES *)GetResWithName(type, res_all.res_changer.hdr.name)) == NULL) { - Emsg1(M_ERROR_TERM, 0, _("Cannot find AutoChanger resource %s\n"), - res_all.res_changer.hdr.name); + Mmsg(config->m_errmsg, _("Cannot find AutoChanger resource %s\n"), + res_all.res_changer.hdr.name); + return false; } /* we must explicitly copy the device alist pointer */ res->res_changer.device = res_all.res_changer.device; @@ -627,13 +873,19 @@ void save_resource(int type, RES_ITEM *items, int pass) foreach_alist(dev, res->res_changer.device) { dev->changer_res = (AUTOCHANGER *)&res->res_changer; } - if ((errstat = rwl_init(&res->res_changer.changer_lock, - PRIO_SD_ACH_ACCESS)) != 0) - { + if ((errstat = rwl_init(&res->res_changer.changer_lock, PRIO_SD_ACH_ACCESS)) != 0) { berrno be; - Jmsg1(NULL, M_ERROR_TERM, 0, _("Unable to init lock: ERR=%s\n"), - be.bstrerror(errstat)); + Mmsg(config->m_errmsg, _("Unable to init lock for Autochanger=%s: ERR=%s\n"), + res_all.res_changer.hdr.name, be.bstrerror(errstat)); + return false; + } + break; + case R_DEVICE: + if ((res = (URES *)GetResWithName(R_DEVICE, res_all.res_dev.hdr.name)) == NULL) { + Mmsg(config->m_errmsg, _("Cannot find Device resource %s\n"), res_all.res_dir.hdr.name); + return false; } + res->res_dev.cloud = res_all.res_dev.cloud; break; default: printf(_("Unknown resource type %d\n"), type); @@ -650,7 +902,7 @@ void save_resource(int type, RES_ITEM *items, int pass) free(res_all.res_dir.hdr.desc); res_all.res_dir.hdr.desc = NULL; } - return; + return true; } /* The following code is only executed on pass 1 */ @@ -670,6 +922,9 @@ void save_resource(int type, RES_ITEM *items, int pass) case R_AUTOCHANGER: size = sizeof(AUTOCHANGER); break; + case R_CLOUD: + size = sizeof(CLOUD); + break; default: printf(_("Unknown resource type %d\n"), type); error = 1; @@ -678,31 +933,16 @@ void save_resource(int type, RES_ITEM *items, int pass) } /* Common */ if (!error) { - res = (URES *)malloc(size); - memcpy(res, &res_all, size); - if (!res_head[rindex]) { - res_head[rindex] = (RES *)res; /* store first entry */ - } else { - RES *next, *last; - /* Add new res to end of chain */ - for (last=next=res_head[rindex]; next; next=next->next) { - last = next; - if (strcmp(next->name, res->res_dir.hdr.name) == 0) { - Emsg2(M_ERROR_TERM, 0, - _("Attempt to define second \"%s\" resource named \"%s\" is not permitted.\n"), - resources[rindex].name, res->res_dir.hdr.name); - } - } - last->next = (RES *)res; - Dmsg2(90, "Inserting %s res: %s\n", res_to_str(type), - res->res_dir.hdr.name); + if (!config->insert_res(rindex, size)) { + return false; } } + return true; } bool parse_sd_config(CONFIG *config, const char *configfile, int exit_code) { config->init(configfile, NULL, exit_code, (void *)&res_all, res_all_size, - r_first, r_last, resources, res_head); + r_first, r_last, resources, &res_head); return config->parse_config(); } diff --git a/bacula/src/stored/stored_conf.h b/bacula/src/stored/stored_conf.h index 9dee7ba687..f081e94649 100644 --- a/bacula/src/stored/stored_conf.h +++ b/bacula/src/stored/stored_conf.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -19,19 +19,39 @@ extern s_kw dev_types[]; +/* + * Cloud Truncate Cache options + */ +enum { + TRUNC_NO = 0, /* default value */ + TRUNC_AFTER_UPLOAD = 1, + TRUNC_AT_ENDOFJOB = 2 +}; + +/* + * Cloud Upload options + */ +enum { + UPLOAD_EACHPART = 0, /* default value */ + UPLOAD_NO = 1, + UPLOAD_AT_ENDOFJOB = 2 +}; + + /* * Resource codes -- they must be sequential for indexing * */ enum { - R_DIRECTOR = 3001, - R_STORAGE, - R_DEVICE, - R_MSGS, - R_AUTOCHANGER, + R_DIRECTOR = 3001, + R_STORAGE = 3002, + R_DEVICE = 3003, + R_MSGS = 3004, + R_AUTOCHANGER = 3005, + R_CLOUD = 3006, R_FIRST = R_DIRECTOR, - R_LAST = R_AUTOCHANGER /* keep this updated */ + R_LAST = R_CLOUD /* keep this updated */ }; enum { @@ -44,6 +64,32 @@ enum { /* Definition of the contents of each Resource */ + +/* + * Cloud drivers + */ +class CLOUD { +public: + RES hdr; + char *host_name; + char *bucket_name; + char *access_key; + char *secret_key; + char *region; + int32_t protocol; + int32_t uri_style; + uint32_t driver_type; /* Cloud driver type */ + uint32_t trunc_opt; + uint32_t upload_opt; + uint32_t max_concurrent_uploads; + uint32_t max_concurrent_downloads; + uint64_t upload_limit; + uint64_t download_limit; +}; + +/* + * Director resource + */ class DIRRES { public: RES hdr; @@ -83,6 +129,7 @@ public: utime_t ClientConnectTimeout; /* Max time to wait to connect client */ utime_t heartbeat_interval; /* Interval to send hb to FD */ utime_t client_wait; /* Time to wait for FD to connect */ + bool comm_compression; /* Set to allow comm line compression */ bool tls_authenticate; /* Authenticate with TLS */ bool tls_enable; /* Enable TLS */ bool tls_require; /* Require TLS */ @@ -105,6 +152,7 @@ public: alist *device; /* List of DEVRES device pointers */ char *changer_name; /* Changer device name */ char *changer_command; /* Changer command -- external program */ + char *lock_command; /* Share storage lock command -- external program */ brwlock_t changer_lock; /* One changer operation at a time */ }; @@ -115,10 +163,12 @@ public: char *media_type; /* User assigned media type */ char *device_name; /* Archive device name */ + char *adevice_name; /* Aligned device name */ char *changer_name; /* Changer device name */ char *control_name; /* SCSI control device name */ char *changer_command; /* Changer command -- external program */ char *alert_command; /* Alert command -- external program */ + char *lock_command; /* Share storage lock command -- external program */ char *spool_directory; /* Spool file directory */ uint32_t dev_type; /* device type */ uint32_t label_type; /* label type */ @@ -132,6 +182,7 @@ public: utime_t max_open_wait; /* maximum secs to wait for open */ uint32_t padding_size; /* adata block padding -- bytes */ uint32_t file_alignment; /* adata file alignment -- bytes */ + uint32_t min_aligned_size; /* minimum adata size */ uint32_t min_block_size; /* min block size */ uint32_t max_block_size; /* max block size */ uint32_t max_volume_jobs; /* max jobs to put on one volume */ @@ -152,18 +203,19 @@ public: char *unmount_command; /* Unmount command */ char *write_part_command; /* Write part command */ char *free_space_command; /* Free space command */ + CLOUD *cloud; /* pointer to cloud resource */ /* The following are set at runtime */ DEVICE *dev; /* Pointer to phyical dev -- set at runtime */ AUTOCHANGER *changer_res; /* pointer to changer res if any */ }; - union URES { DIRRES res_dir; STORES res_store; DEVRES res_dev; MSGS res_msgs; AUTOCHANGER res_changer; + CLOUD res_cloud; RES hdr; }; diff --git a/bacula/src/stored/tape_alert.c b/bacula/src/stored/tape_alert.c new file mode 100644 index 0000000000..0cf12beaac --- /dev/null +++ b/bacula/src/stored/tape_alert.c @@ -0,0 +1,230 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * Routines for getting and displaying tape alerts + * + * Written by Kern Sibbald, October MMXVI + * + */ + +#include "bacula.h" /* pull in global headers */ +#include "stored.h" /* pull in Storage Deamon headers */ + +#include "tape_alert_msgs.h" + +static const int dbglvl = 120; + +#define MAX_MSG 54 /* Maximum alert message number */ + +void alert_callback(void *ctx, const char *short_msg, const char *long_msg, + char *Volume, int severity, int flags, int alertno, utime_t alert_time) +{ + DCR *dcr = (DCR *)ctx; + JCR *jcr = dcr->jcr; + DEVICE *dev = dcr->dev; + int type = M_INFO; + + switch (severity) { + case 'C': + type = M_FATAL; + break; + case 'W': + type = M_WARNING; + break; + case 'I': + type = M_INFO; + break; + } + if (flags & TA_DISABLE_DRIVE) { + dev->enabled = false; + Jmsg(jcr, M_WARNING, 0, _("Disabled Device %s due to tape alert=%d.\n"), + dev->print_name(), alertno); + Tmsg2(dbglvl, _("Disabled Device %s due to tape alert=%d.\n"), + dev->print_name(), alertno); + } + if (flags & TA_DISABLE_VOLUME) { + dev->setVolCatStatus("Disabled"); + dev->VolCatInfo.VolEnabled = false; + dir_update_volume_info(dcr, false, true); + Jmsg(jcr, M_WARNING, 0, _("Disabled Volume \"%s\" due to tape alert=%d.\n"), + Volume, alertno); + Tmsg2(dbglvl, _("Disabled Volume \"%s\" due to tape alert=%d.\n"), + Volume, alertno); + } + Jmsg(jcr, type, (utime_t)alert_time, _("Alert: Volume=\"%s\" alert=%d: ERR=%s\n"), + Volume, alertno, long_msg); +} + +bool tape_dev::get_tape_alerts(DCR *dcr) +{ + JCR *jcr = dcr->jcr; + + if (!job_canceled(jcr) && dcr->device->alert_command && + dcr->device->control_name) { + POOLMEM *alertcmd; + int status = 1; + int nalerts = 0; + BPIPE *bpipe; + ALERT *alert, *rmalert; + char line[MAXSTRING]; + const char *fmt = "TapeAlert[%d]"; + + if (!alert_list) { + alert_list = New(alist(10)); + } + alertcmd = get_pool_memory(PM_FNAME); + alertcmd = edit_device_codes(dcr, alertcmd, dcr->device->alert_command, ""); + /* Wait maximum 5 minutes */ + bpipe = open_bpipe(alertcmd, 60 * 5, "r"); + if (bpipe) { + int alertno; + alert = (ALERT *)malloc(sizeof(ALERT)); + memset(alert->alerts, 0, sizeof(alert->alerts)); + alert->Volume = bstrdup(getVolCatName()); + alert->alert_time = (utime_t)time(NULL); + while (fgets(line, (int)sizeof(line), bpipe->rfd)) { + alertno = 0; + if (bsscanf(line, fmt, &alertno) == 1) { + if (alertno > 0) { + if (nalerts+1 > (int)sizeof(alert->alerts)) { + break; + } else { + alert->alerts[nalerts++] = alertno; + } + } + } + } + status = close_bpipe(bpipe); + if (nalerts > 0) { + /* Maintain First in, last out list */ + if (alert_list->size() > 8) { + rmalert = (ALERT *)alert_list->last(); + free(rmalert->Volume); + alert_list->pop(); + free(rmalert); + } + alert_list->prepend(alert); + } else { + free(alert->Volume); + free(alert); + } + free_pool_memory(alertcmd); + return true; + } else { + status = errno; + } + if (status != 0) { + berrno be; + Jmsg(jcr, M_ALERT, 0, _("3997 Bad alert command: %s: ERR=%s.\n"), + alertcmd, be.bstrerror(status)); + Tmsg2(10, _("3997 Bad alert command: %s: ERR=%s.\n"), + alertcmd, be.bstrerror(status)); + } + + Dmsg1(400, "alert status=%d\n", status); + free_pool_memory(alertcmd); + } else { + if (!dcr->device->alert_command) { + Dmsg1(dbglvl, "Cannot do tape alerts: no Alert Command specified for device %s\n", + print_name()); + Tmsg1(dbglvl, "Cannot do tape alerts: no Alert Command specified for device %s\n", + print_name()); + + } + if (!dcr->device->control_name) { + Dmsg1(dbglvl, "Cannot do tape alerts: no Control Device specified for device %s\n", + print_name()); + Tmsg1(dbglvl, "Cannot do tape alerts: no Control Device specified for device %s\n", + print_name()); + } + } + return false; +} + + +/* + * Print desired tape alert messages + */ +void tape_dev::show_tape_alerts(DCR *dcr, alert_list_type list_type, + alert_list_which which, alert_cb alert_callback) +{ + int i; + ALERT *alert; + int code; + + if (!alert_list) { + return; + } + Dmsg1(dbglvl, "There are %d alerts.\n", alert_list->size()); + switch (list_type) { + case list_codes: + foreach_alist(alert, alert_list) { + for (i=0; i<(int)sizeof(alert->alerts) && alert->alerts[i]; i++) { + code = alert->alerts[i]; + Dmsg4(dbglvl, "Volume=%s alert=%d severity=%c flags=0x%x\n", alert->Volume, code, + ta_errors[code].severity, (int)ta_errors[code].flags); + alert_callback(dcr, ta_errors[code].short_msg, long_msg[code], + alert->Volume, ta_errors[code].severity, + ta_errors[code].flags, code, (utime_t)alert->alert_time); + } + if (which == list_last) { + break; + } + } + break; + default: + foreach_alist(alert, alert_list) { + for (i=0; i<(int)sizeof(alert->alerts) && alert->alerts[i]; i++) { + code = alert->alerts[i]; + Dmsg4(dbglvl, "Volume=%s severity=%c flags=0x%x alert=%s\n", alert->Volume, + ta_errors[code].severity, (int)ta_errors[code].flags, + ta_errors[code].short_msg); + alert_callback(dcr, ta_errors[code].short_msg, long_msg[code], + alert->Volume, ta_errors[code].severity, + ta_errors[code].flags, code, (utime_t)alert->alert_time); + } + if (which == list_last) { + break; + } + } + break; + } + return; +} + +/* + * Delete alert list returning number deleted + */ +int tape_dev::delete_alerts() +{ + ALERT *alert; + int deleted = 0; + + if (alert_list) { + foreach_alist(alert, alert_list) { + free(alert->Volume); + deleted++; + } + alert_list->destroy(); + free(alert_list); + alert_list = NULL; + } + return deleted; +} diff --git a/bacula/src/stored/tape_alert_msgs.h b/bacula/src/stored/tape_alert_msgs.h new file mode 100644 index 0000000000..45ab2f0367 --- /dev/null +++ b/bacula/src/stored/tape_alert_msgs.h @@ -0,0 +1,166 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ +/* + * + * Routines for getting and displaying tape alerts + * + * Written by Kern Sibbald, October MMXVI + * + */ + +struct ta_error_handling { + const char severity; + const char flags; + const char *short_msg; +}; + +#define TA_NONE (0) +#define TA_DISABLE_DRIVE (1<<0) +#define TA_DISABLE_VOLUME (1<<1) +#define TA_CLEAN_DRIVE (1<<2) +#define TA_PERIODIC_CLEAN (1<<3) +#define TA_RETENTION (1<<4) + +/* + * ta_error_handling determines error handling + * Severity determines if we have info, warning or fail (critical) error. + * Flags allows setting flags + * + */ +static struct ta_error_handling ta_errors[] = { + {' ', TA_NONE, ""}, +/* 1 */ {'W', TA_NONE, "Read Warning"}, + {'W', TA_NONE, "Write Warning"}, + {'C', TA_NONE, "Hard Error"}, + {'C', TA_NONE, "Media"}, + {'C', TA_NONE, "Read Failure"}, + {'C', TA_NONE, "Write Failure"}, + {'W', TA_DISABLE_VOLUME, "Media Life"}, + {'W', TA_DISABLE_VOLUME, "Not Data Grade"}, + {'C', TA_NONE, "Write Protect"}, +/* 10 */ {'I', TA_NONE, "No Removal"}, + {'I', TA_NONE, "Cleaning Media"}, + {'I', TA_NONE, "Unsupported Format"}, + {'C', TA_DISABLE_VOLUME, "Recoverable Snapped Tape"}, + {'C', TA_DISABLE_DRIVE| \ + TA_DISABLE_VOLUME, "Unrecoverable Snapped Tape"}, + {'W', TA_NONE, "Cartridge Memory Chip Failure"}, + {'C', TA_NONE, "Forced Eject"}, + {'W', TA_NONE, "Read Only Format"}, + {'W', TA_NONE, "Tape Directory Corrupted on load"}, + {'I', TA_NONE, "Nearing Media Life"}, +/* 20 */ {'C', TA_CLEAN_DRIVE| \ + TA_DISABLE_DRIVE| \ + TA_DISABLE_VOLUME, "Clean Now"}, + {'W', TA_PERIODIC_CLEAN, "Clean Periodic"}, + {'C', TA_DISABLE_VOLUME, "Expired Cleaning Media"}, + {'C', TA_RETENTION, "Invalid Cleaning Media"}, + {'W', TA_NONE, "Retention Requested"}, + {'W', TA_NONE, "Dual-Port Interface Error"}, + {'W', TA_NONE, "Cooling Fan Failure"}, + {'W', TA_NONE, "Power Supply Failure"}, + {'W', TA_NONE, "Power Consumption"}, + {'W', TA_DISABLE_DRIVE, "Drive Maintenance"}, +/* 30 */ {'C', TA_DISABLE_DRIVE, "Hardware A"}, + {'C', TA_DISABLE_DRIVE, "Hardware B"}, + {'W', TA_NONE, "Interface"}, + {'C', TA_NONE, "Eject Media"}, + {'W', TA_NONE, "Download Fail"}, + {'W', TA_NONE, "Drive Humidity"}, + {'W', TA_NONE, "Drive Temperature"}, + {'W', TA_NONE, "Drive Voltage"}, + {'C', TA_DISABLE_DRIVE, "Predictive Failure"}, + {'W', TA_DISABLE_DRIVE, "Diagnostics Required"}, +/* 40 */ {'C', TA_NONE, "Loader Hardware A"}, + {'C', TA_NONE, "Loader Stray Tape"}, + {'W', TA_NONE, "Loader Hardware B"}, + {'C', TA_NONE, "Loader Door"}, + {'C', TA_NONE, "Loader Hardware C"}, + {'C', TA_NONE, "Loader Magazine"}, +/* 46 */ {'W', TA_NONE, "Loader Predictive Failure"}, + {' ', TA_NONE, ""}, + {' ', TA_NONE, ""}, + {' ', TA_NONE, ""}, +/* 50 */ {'W', TA_NONE, "Lost Statistics"}, + {'W', TA_NONE, "Tape directory invalid at unload"}, + {'C', TA_DISABLE_VOLUME, "Tape system area write failure"}, + {'C', TA_DISABLE_VOLUME, "Tape system area read failure"}, +/* 54 */ {'C', TA_DISABLE_VOLUME, "No start of data"} +}; + +/* + * Long message, sometimes even too verbose. + */ +static const char *long_msg[] = { + "", +/* 1 */ "The tape drive is having problems reading data. No data has been lost, but there has been a reduction in the performance of the tape. The drive is having severe trouble reading", + "The tape drive is having problems writing data. No data has been lost, but there has been a reduction in the capacity of the tape. The drive is having severe trouble writing", + "The operation has stopped because an error has occurred while reading or writing data which the drive cannot correct. The drive had a hard read or write error", + "Your data is at risk: Media cannot be written/read, or media performance is severely degraded.\n 1. Copy any data you require from the tape.\n 2. Do not use this tape again.\n 3. Restart the operation with a different tape.", + "The tape is damaged or the drive is faulty. Call the tape drive supplier helpline. The drive can no longer read data from the tape", + "The tape is from a faulty batch or the tape drive is faulty: The drive can no longer write data to the tape.\n 1. Use a good tape to test the drive.\n 2. If the problem persists, call the tape drive supplier helpline.", + "The tape cartridge has reached the end of its calculated useful life: The media has exceeded its specified life.\n 1. Copy any data you need to another tape.\n 2. Discard the old tape.", + "The tape cartridge is not data-grade. Any data you back up to the tape is at risk. The drive has not been able to read the MRS stripes. Replace the cartridge with a data-grade tape.", + "You are trying to write to a write-protected cartridge. Write command is attempted to a write protected tape. Remove the write-protection or use another tape.", +/* 10 */ "You cannot eject the cartridge because the tape drive is in use. Manual or s/w unload attempted when prevent media removal is enabled. Wait until the operation is complete before ejecting the cartridge.", + "The tape in the drive is a cleaning cartridge. Cleaning tape loaded in drive.", + "You have tried to load a cartridge of a type which is not supported by this drive. Attempted loaded of unsupported tape format, e.g. DDS2 in DDS1 drive.", + "The operation has failed because the tape in the drive has snapped: Tape snapped/cut in the drive where media can be ejected.\n 1. Discard the old tape.\n 2. Restart the operation with a different tape.", + "The operation has failed because the tape in the drive has snapped: Tape snapped/cut in the drive where media cannot be ejected.\n 1. Do not attempt to extract the tape cartridge.\n 2. Call the tape drive supplier helpline.", + "The memory in the tape cartridge has failed, which reduces performance. Memory chip failed in cartridge. Do not use the cartridge for further backup operations.", + "The operation has failed because the tape cartridge was manually ejected while the tape drive was actively writing or reading. Manual or forced eject while drive actively writing or reading", + "You have loaded a cartridge of a type that is read-only in this drive. Media loaded that is read-only format. The cartridge will appear as write-protected.", + "The directory on the tape cartridge has been corrupted. Tape drive powered down with tape loaded, or permanent error prevented the tape directory being updated. File search performance will be degraded. The tape directory can be rebuilt by reading all the data on the cartridge.", + "The tape cartridge is nearing the end of its calculated life. Media may have exceeded its specified number of passes. It is recommended that you:\n 1. Use another tape cartridge for your next backup.\n 2. Store this tape cartridge in a safe place in case you need to restore data from it.", +/* 20 */ "The tape drive needs cleaning: The drive thinks it has a head clog, or needs cleaning.\n 1. If the operation has stopped, eject the tape and clean the drive.\n 2. If the operation has not stopped, wait for it to finish and then clean the drive.\n Check the tape drive users manual for device specific cleaning instructions.", + "The tape drive is due for routine cleaning: The drive is ready for a periodic clean.\n 1. Wait for the current operation to finish.\n 2. Then use a cleaning cartridge.\n Check the tape drive users manual for device specific cleaning instructions.", + "The last cleaning cartridge used in the tape drive has worn out. The cleaning tape has expired.\n 1. Discard the worn out cleaning cartridge.\n 2. Wait for the current operation to finish.\n 3. Then use a new cleaning cartridge.", + "The last cleaning cartridge used in the tape drive was an invalid type: Invalid cleaning tape type used.\n 1. Do not use this cleaning cartridge in this drive.\n 2. Wait for the current operation to finish.\n 3. Then use a valid cleaning cartridge.", + "The tape drive has requested a retention operation. The drive is having severe trouble reading or writing, which will be resolved by a retention cycle.", + "A redundant interface port on the tape drive has failed. Failure of one interface port in a dual-port configuration, e.g. Fibrechannel.", + "A tape drive cooling fan has failed. Fan failure inside tape drive mechanism or tape drive enclosure.", + "A redundant power supply has failed inside the tape drive enclosure. Check the enclosure users manual for instructions on replacing the failed power supply. Redundant PSU failure inside the tape drive enclosure or rack subsystem.", + "The tape drive power consumption is outside the specified range. Power consumption of the tape drive is outside specified range.", + "Preventive maintenance of the tape drive is required. The drive requires preventative maintenance (not cleaning). Check the tape drive users manual for device specific preventive maintenance tasks or call the tape drive supplier helpline.", +/* 30 */ "The tape drive has a hardware fault: The drive has a hardware fault that requires reset to recover.\n 1. Eject the tape or magazine.\n 2. Reset the drive.\n 3. Restart the operation.", + "The tape drive has a hardware fault: The drive has a hardware fault which is not read/write related or requires a power cycle to recover.\n 1. Turn the tape drive off and then on again.\n 2. Restart the operation.\n 3. If the problem persists, call the tape drive supplier helpline.\n Check the tape drive users manual for device specific instructions on turning the device power on and off.", + "The tape drive has a problem with the host interface: The drive has identified an interfacing fault.\n 1. Check the cables and cable connections.\n 2. Restart the operation.", + "The operation has failed. Error recovery action:\n 1. Eject the tape or magazine.\n 2. Insert the tape or magazine again.\n 3. Restart the operation.", + "The firmware download has failed because you have tried to use the incorrect firmware for this tape drive. Firmware download failed. Obtain the correct firmware and try again.", + "Environmental conditions inside the tape drive are outside the specified humidity range. Drive humidity limits exceeded.", + "Environmental conditions inside the tape drive are outside the specified temperature range. Drive temperature limits exceeded.", + "The voltage supply to the tape drive is outside the specified range. Drive voltage limits exceeded.", + "A hardware failure of the tape drive is predicted. Call the tape drive supplier helpline. Predictive failure of drive hardware.", + "The tape drive may have a fault. Check for availability of diagnostic information and run extended diagnostics if applicable. The drive may have had a failure which may be identified by stored diagnostic information or by running extended diagnostics (eg Send Diagnostic). Check the tape drive users manual for instructions on running extended diagnostic tests and retrieving diagnostic data.", +/* 40 */ "The changer mechanism is having difficulty communicating with the tape drive: Loader mech. is having trouble communicating with the tape drive.\n 1. Turn the autoloader off then on.\n 2. Restart the operation.\n 3. If problem persists, call the tape drive supplier helpline.", + "A tape has been left in the autoloader by a previous hardware fault: Stray tape left in loader after pervious error recovery.\n 1. Insert an empty magazine to clear the fault.\n 2. If the fault does not clear, turn the autoloader off and then on again.\n 3. If the problem persists, call the tape drive supplier helpline.", + "There is a problem with the autoloader mechanism. Loader mech. has a hardware fault.", + "The operation has failed because the autoloader door is open: Tape changer door open:\n 1. Clear any obstructions from the autoloader door.\n 2. Eject the magazine and then insert it again.\n 3. If the fault does not clear, turn the autoloader off and then on again.\n 4. If the problem persists, call the tape drive supplier helpline.", + "The autoloader has a hardware fault: The loader mechanism has a hardware fault that is not mechanically related.\n 1. Turn the autoloader off and then on again.\n 2. Restart the operation.\n 3. If the problem persists, call the tape drive supplier helpline.\n Check the autoloader users manual for device specific instructions on turning the device power on and off.", + "The autoloader cannot operate without the magazine. Loader magazine not present.\n 1. Insert the magazine into the autoloader.\n 2. Restart the operation.", +/* 46 */ "A hardware failure of the changer mechanism is predicted. Predictive failure of loader mechanism hardware. Call the tape drive supplier helpline.", + "", + "", + "", +/* 50 */ "Media statistics have been lost at some time in the past, Drive or library powered down with tape loaded.", + "The tape directory on the tape cartridge just unloaded has been corrupted. Error prevented the tape directory being updated on unload. File search performance will be degraded. The tape directory can be rebuilt by reading all the data.", + "The tape just unloaded could not write its system area successfully: Write errors while writing the system log on unload.\n 1. Copy data to another tape cartridge.\n 2. Discard the old cartridge.", + "The tape system area could not be read successfully at load time: Read errors while reading the system area on load.\n 1. Copy data to another tape cartridge.\n 2. Discard the old cartridge.", +/* 54 */ "The start of data could not be found on the tape: Tape damaged, bulk erased, or incorrect format.\n 1. Check you are using the correct format tape.\n 2. Discard the tape or return the tape to your supplier." +}; diff --git a/bacula/src/stored/tape_dev.c b/bacula/src/stored/tape_dev.c index 212819a3ca..908149a8d3 100644 --- a/bacula/src/stored/tape_dev.c +++ b/bacula/src/stored/tape_dev.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -70,7 +70,7 @@ const char *mode_to_str(int mode); /* */ -void DEVICE::open_tape_device(DCR *dcr, int omode) +bool tape_dev::open_device(DCR *dcr, int omode) { file_size = 0; int timeout = max_open_wait; @@ -79,7 +79,12 @@ void DEVICE::open_tape_device(DCR *dcr, int omode) utime_t start_time = time(NULL); #endif - mount(1); /* do mount if required */ + if (DEVICE::open_device(dcr, omode)) { + return true; /* already open */ + } + omode = openmode; /* pickup possible new options */ + + mount(1); /* do mount if required */ Dmsg0(100, "Open dev: device is tape\n"); @@ -97,6 +102,7 @@ void DEVICE::open_tape_device(DCR *dcr, int omode) tid = start_thread_timer(dcr->jcr, pthread_self(), timeout); } Dmsg2(100, "Try open %s mode=%s\n", print_name(), mode_to_str(omode)); + #if defined(HAVE_WIN32) /* Windows Code */ @@ -174,6 +180,8 @@ void DEVICE::open_tape_device(DCR *dcr, int omode) tid = 0; } Dmsg1(100, "open dev: tape %d opened\n", m_fd); + state |= preserve; /* reset any important state info */ + return m_fd >= 0; } @@ -220,7 +228,7 @@ bool tape_dev::rewind(DCR *dcr) int open_mode = openmode; d_close(m_fd); clear_opened(); - open(dcr, open_mode); + open_device(dcr, open_mode); if (m_fd < 0) { return false; } @@ -249,23 +257,57 @@ bool tape_dev::rewind(DCR *dcr) return true; } +/* + * Check if the current position on the volume corresponds to + * what is in the catalog. + * + */ +bool tape_dev::is_eod_valid(DCR *dcr) +{ + JCR *jcr = dcr->jcr; + /* + * Check if we are positioned on the tape at the same place + * that the database says we should be. + */ + if (VolCatInfo.VolCatFiles == get_file()) { + Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volume \"%s\" at file=%d.\n"), + dcr->VolumeName, get_file()); + } else if (get_file() > VolCatInfo.VolCatFiles) { + Jmsg(jcr, M_WARNING, 0, _("For Volume \"%s\":\n" + "The number of files mismatch! Volume=%u Catalog=%u\n" + "Correcting Catalog\n"), + dcr->VolumeName, get_file(), VolCatInfo.VolCatFiles); + VolCatInfo.VolCatFiles = get_file(); + VolCatInfo.VolCatBlocks = get_block_num(); + if (!dir_update_volume_info(dcr, false, true)) { + Jmsg(jcr, M_WARNING, 0, _("Error updating Catalog\n")); + dcr->mark_volume_in_error(); + return false; + } + } else { + Jmsg(jcr, M_ERROR, 0, _("Bacula cannot write on tape Volume \"%s\" because:\n" + "The number of files mismatch! Volume=%u Catalog=%u\n"), + dcr->VolumeName, get_file(), VolCatInfo.VolCatFiles); + dcr->mark_volume_in_error(); + return false; + } + return true; +} + /* * Position device to end of medium (end of data) * Returns: true on succes * false on error */ -bool DEVICE::eod(DCR *dcr) +bool tape_dev::eod(DCR *dcr) { struct mtop mt_com; bool ok = true; - boffset_t pos; int32_t os_file; Enter(100); - if (m_fd < 0) { - dev_errno = EBADF; - Mmsg1(errmsg, _("Bad call to eod. Device %s not open\n"), print_name()); - Dmsg1(100, "%s", errmsg); + ok = DEVICE::eod(dcr); + if (!ok) { return false; } @@ -273,41 +315,12 @@ bool DEVICE::eod(DCR *dcr) return fsf(VolCatInfo.VolCatFiles); #endif - if (at_eot()) { - Leave(100); - return true; - } - clear_eof(); /* remove EOF flag */ - block_num = file = 0; - file_size = 0; - file_addr = 0; - if (is_fifo()) { - Leave(100); - return true; - } - if (!is_tape()) { - pos = lseek(dcr, (boffset_t)0, SEEK_END); - Dmsg1(200, "====== Seek to %lld\n", pos); - if (pos >= 0) { - update_pos(dcr); - set_eot(); - Leave(100); - return true; - } - dev_errno = errno; - berrno be; - Mmsg2(errmsg, _("lseek error on %s. ERR=%s.\n"), - print_name(), be.bstrerror()); - Dmsg1(100, "%s", errmsg); - Leave(100); - return false; - } #ifdef MTEOM if (has_cap(CAP_FASTFSF) && !has_cap(CAP_EOM)) { Dmsg0(100,"Using FAST FSF for EOM\n"); /* If unknown position, rewind */ if (get_os_tape_file() < 0) { - if (!rewind(NULL)) { + if (!rewind(dcr)) { Dmsg0(100, "Rewind error\n"); Leave(100); return false; @@ -363,7 +376,7 @@ bool DEVICE::eod(DCR *dcr) /* * Rewind then use FSF until EOT reached */ - if (!rewind(NULL)) { + if (!rewind(dcr)) { Dmsg0(100, "Rewind error.\n"); Leave(100); return false; @@ -468,7 +481,7 @@ bool load_dev(DEVICE *dev) * Returns: true on success * false on failure */ -bool tape_dev::offline() +bool tape_dev::offline(DCR *dcr) { struct mtop mt_com; @@ -494,13 +507,13 @@ bool tape_dev::offline() return true; } -bool DEVICE::offline_or_rewind() +bool DEVICE::offline_or_rewind(DCR *dcr) { if (m_fd < 0) { return false; } if (has_cap(CAP_OFFLINEUNMOUNT)) { - return offline(); + return offline(dcr); } else { /* * Note, this rewind probably should not be here (it wasn't @@ -510,7 +523,7 @@ bool DEVICE::offline_or_rewind() * done, all future references to the drive get and I/O error. */ clrerror(MTREW); - return rewind(NULL); + return rewind(dcr); } } @@ -849,8 +862,12 @@ void tape_dev::unlock_door() * Returns: false on failure * true on success */ -bool tape_dev::reposition(DCR *dcr, uint32_t rfile, uint32_t rblock) +bool tape_dev::reposition(DCR *dcr, uint64_t raddr) { + uint32_t rfile, rblock; + + rfile = (uint32_t)(raddr>>32); + rblock = (uint32_t)raddr; if (!is_open()) { dev_errno = EBADF; Mmsg0(errmsg, _("Bad call to reposition. Device not open\n")); @@ -862,7 +879,7 @@ bool tape_dev::reposition(DCR *dcr, uint32_t rfile, uint32_t rblock) Dmsg4(100, "reposition from %u:%u to %u:%u\n", file, block_num, rfile, rblock); if (rfile < file) { Dmsg0(100, "Rewind\n"); - if (!rewind(NULL)) { + if (!rewind(dcr)) { return false; } } @@ -906,7 +923,7 @@ bool tape_dev::reposition(DCR *dcr, uint32_t rfile, uint32_t rblock) * Returns: true on success * false on failure */ -bool DEVICE::weof(int num) +bool tape_dev::weof(DCR *dcr, int num) { struct mtop mt_com; int stat; @@ -946,6 +963,12 @@ bool DEVICE::weof(int num) print_name(), be.bstrerror()); } } + /* DCR is null if called from within write_ansi_ibm_labels() */ + if (dcr && stat == 0) { + if (!write_ansi_ibm_labels(dcr, ANSI_EOF_LABEL, VolHdr.VolumeName)) { + stat = -1; + } + } return stat == 0; } @@ -1033,3 +1056,63 @@ bool tape_dev::mount_tape(int mount, int dotimeout) Dmsg1(200, "============ mount=%d\n", mount); return true; } + +void tape_dev::set_ateof() +{ + DEVICE::set_ateof(); + file++; +} + +const char *tape_dev::print_type() +{ + return "Tape"; +} + +DEVICE *tape_dev::get_dev(DCR */*dcr*/) +{ + return this; +} + +uint32_t tape_dev::get_hi_addr() +{ + return file; +} + +uint32_t tape_dev::get_low_addr() +{ + return block_num; +} + +uint64_t tape_dev::get_full_addr() +{ + return (((uint64_t)file) << 32) | (uint64_t)block_num; +} + +bool tape_dev::end_of_volume(DCR *dcr) +{ + return write_ansi_ibm_labels(dcr, ANSI_EOV_LABEL, VolHdr.VolumeName); +} + +/* Print the address */ +char *tape_dev::print_addr(char *buf, int32_t buf_len) +{ + buf[0] = 0; + bsnprintf(buf, buf_len, "%lu:%lu", get_hi_addr(), get_low_addr()); + return buf; +} + +char *tape_dev::print_addr(char *buf, int32_t buf_len, boffset_t addr) +{ + buf[0] = 0; + bsnprintf(buf, buf_len, "%lu:%lu", (uint32_t)(addr>>32), (uint32_t)addr); + return buf; +} + +/* + * Clean up when terminating the device + */ +void tape_dev::term(DCR *dcr) +{ + delete_alerts(); + DEVICE::term(dcr); +} diff --git a/bacula/src/stored/tape_dev.h b/bacula/src/stored/tape_dev.h index 557e340d25..b6d9a74a78 100644 --- a/bacula/src/stored/tape_dev.h +++ b/bacula/src/stored/tape_dev.h @@ -1,7 +1,7 @@ -/* [vssfs.c] IQ +/* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -23,6 +23,12 @@ #ifndef __TAPE_DEV_ #define __TAPE_DEV_ +struct ALERT { + char *Volume; + utime_t alert_time; + char alerts[10]; +}; + class tape_dev : public DEVICE { public: @@ -31,15 +37,35 @@ public: /* DEVICE virtual functions that we redefine with our tape code */ bool fsf(int num); - bool offline(); + bool offline(DCR *dcr); bool rewind(DCR *dcr); bool bsf(int num); void lock_door(); void unlock_door(); - bool reposition(DCR *dcr, uint32_t rfile, uint32_t rblock); + bool reposition(DCR *dcr, uint64_t raddr); bool mount(int timeout); bool unmount(int timeout); bool mount_tape(int mount, int dotimeout); + bool weof(DCR *dcr, int num); + bool eod(DCR *dcr); + bool is_eod_valid(DCR *dcr); + void set_ateof(); + bool open_device(DCR *dcr, int omode); + void term(DCR *dcr); + const char *print_type(); + DEVICE *get_dev(DCR *dcr); + uint32_t get_hi_addr(); + uint32_t get_low_addr(); + uint64_t get_full_addr(); + bool end_of_volume(DCR *dcr); + char *print_addr(char *buf, int32_t buf_len); + char *print_addr(char *buf, int32_t maxlen, boffset_t addr); + bool get_tape_alerts(DCR *dcr); + void show_tape_alerts(DCR *dcr, alert_list_type type, + alert_list_which which, alert_cb alert_callback); + int delete_alerts(); + + alist *alert_list; }; #endif /* __TAPE_DEV_ */ diff --git a/bacula/src/stored/vbackup.c b/bacula/src/stored/vbackup.c index 49e15c2364..f43632ef52 100644 --- a/bacula/src/stored/vbackup.c +++ b/bacula/src/stored/vbackup.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -63,8 +63,10 @@ bool do_vbackup(JCR *jcr) break; } + /* TODO: Remove when the new match_all is well tested */ + jcr->use_new_match_all = use_new_match_all; - Dmsg0(20, "Start read data.\n"); + Dmsg1(20, "Start read data. newbsr=%d\n", jcr->use_new_match_all); if (!jcr->read_dcr || !jcr->dcr) { Jmsg(jcr, M_FATAL, 0, _("Read and write devices not properly initialized.\n")); @@ -88,6 +90,7 @@ bool do_vbackup(JCR *jcr) jcr->setJobStatus(JS_ErrorTerminated); goto bail_out; } + jcr->dcr->dev->start_of_job(jcr->dcr); Dmsg2(200, "===== After acquire pos %u:%u\n", jcr->dcr->dev->file, jcr->dcr->dev->block_num); jcr->sendJobStatus(JS_Running); @@ -114,6 +117,13 @@ ok_out: dev = jcr->dcr->dev; Dmsg1(100, "ok=%d\n", ok); if (ok || dev->can_write()) { + if (!dev->flush_before_eos(jcr->dcr)) { + Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), + dev->print_name(), dev->bstrerror()); + Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n")); + //possible_incomplete_job(jcr, last_file_index); + ok = false; + } /* Flush out final ameta partial block of this session */ if (!jcr->dcr->write_final_block_to_device()) { Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), @@ -172,8 +182,8 @@ ok_out: generate_daemon_event(jcr, "JobEnd"); generate_plugin_event(jcr, bsdEventJobEnd); dir->fsend(Job_end, jcr->Job, jcr->JobStatus, jcr->JobFiles, - edit_uint64(jcr->JobBytes, ec1), jcr->JobErrors); - Dmsg4(100, Job_end, jcr->Job, jcr->JobStatus, jcr->JobFiles, ec1); + edit_uint64(jcr->JobBytes, ec1), jcr->JobErrors, jcr->StatusErrMsg); + Dmsg6(100, Job_end, jcr->Job, jcr->JobStatus, jcr->JobFiles, ec1, jcr->JobErrors, jcr->StatusErrMsg); dir->signal(BNET_EOD); /* send EOD to Director daemon */ free_plugins(jcr); /* release instantiated plugins */ @@ -233,6 +243,10 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) rec->FileIndex = jcr->JobFiles; /* set sequential output FileIndex */ } + /* TODO: If user really wants to do rehydrate the data, we should propose + * this option. + */ + /* * Modify record SessionId and SessionTime to correspond to * output. diff --git a/bacula/src/stored/vol_mgr.c b/bacula/src/stored/vol_mgr.c index f4c9cdf6e6..23d55606ab 100644 --- a/bacula/src/stored/vol_mgr.c +++ b/bacula/src/stored/vol_mgr.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -706,7 +706,7 @@ bool volume_unused(DCR *dcr) * where the tapes are or last were. */ Dmsg5(dbglvl, "set not reserved vol=%s slot=%d writers=%d reserves=%d dev=%s\n", - dev->vol->vol_name, dev->vol->get_slot(), dev->num_writers, + dev->vol->vol_name, dev->vol->get_slot(), dev->num_writers, dev->num_reserved(), dev->print_name()); if (dev->is_tape() || dev->is_autochanger()) { return true; @@ -742,7 +742,7 @@ bool free_volume(DEVICE *dev) if (vol->is_writing()) { vol_list->remove(vol); } - Dmsg3(dbglvl, "Remove volume %s slot=%d dev=%s\n", vol->vol_name, + Dmsg3(dbglvl, "Remove volume %s slot=%d dev=%s\n", vol->vol_name, vol->get_slot(), dev->print_name()); free_vol_item(vol); debug_list_volumes("free_volume"); @@ -901,6 +901,8 @@ dlist *dup_vol_list(JCR *jcr) memset(tvol, 0, sizeof(VOLRES)); tvol->vol_name = bstrdup(vol->vol_name); tvol->dev = vol->dev; + tvol->init_mutex(); + tvol->inc_use_count(); nvol = (VOLRES *)temp_vol_list->binary_insert(tvol, name_compare); if (tvol != nvol) { tvol->dev = NULL; /* don't zap dev entry */ diff --git a/bacula/src/stored/vtape_dev.c b/bacula/src/stored/vtape_dev.c index 1f9a03e00d..bdf880eb95 100644 --- a/bacula/src/stored/vtape_dev.c +++ b/bacula/src/stored/vtape_dev.c @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -166,7 +166,7 @@ int vtape::tape_op(struct mtop *mt_com) break; case MTOFFL: /* put tape offline */ - result = offline() ? 0 : -1; + result = offline(NULL) ? 0 : -1; break; case MTRETEN: /* Re-tension tape. */ @@ -754,9 +754,9 @@ int vtape::bsf() * * Put vtape in offline mode */ -bool vtape::offline() +bool vtape::offline(DCR *dcr) { - close(); + close(dcr); atEOF = false; /* End of file */ atEOT = false; /* End of tape */ @@ -887,6 +887,8 @@ int vtape::d_open(const char *pathname, int uflags) struct flock lock; struct stat statp; + ASSERT(!m_shstore || (m_shstore_lock && m_shstore_register)); + if (stat(pathname, &statp) != 0) { fd = -1; Dmsg1(dbglevel, "Can't stat on %s\n", pathname); @@ -895,7 +897,7 @@ int vtape::d_open(const char *pathname, int uflags) fd = ::open("/dev/null", O_RDWR | O_LARGEFILE, 0600); } } else { - fd = ::open(pathname, O_RDWR | O_LARGEFILE, 0600); + fd = ::open(pathname, O_RDWR | O_LARGEFILE | O_CLOEXEC, 0600); } if (fd < 0) { @@ -909,7 +911,7 @@ int vtape::d_open(const char *pathname, int uflags) strcpy(lockfile, pathname); strcat(lockfile, ".l"); - lockfd = ::open(lockfile, O_CREAT | O_RDWR | O_LARGEFILE, 0600); + lockfd = ::open(lockfile, O_CREAT | O_RDWR | O_LARGEFILE | O_CLOEXEC, 0600); if (lockfd < 0) { berrno be; Dmsg2(0, "Unable to open vtape device lock %s ERR=%s\n", lockfile, be.bstrerror()); @@ -971,4 +973,26 @@ void vtape::dump() atEOF, atEOT, atEOD, atBOT); } +const char *vtape::print_type() +{ + return "Vtape"; +} + +#else /*!USE_VTAPE */ + +/* faked constructor and destructor to avoid undefined reference to vtable */ +vtape::vtape() +{ +} + +vtape::~vtape() +{ +} + +/* dummy implementation */ +const char *vtape::print_type() +{ + return "Vtape"; +} + #endif /* ! USE_VTAPE */ diff --git a/bacula/src/stored/vtape_dev.h b/bacula/src/stored/vtape_dev.h index 347e182424..d0db3211f0 100644 --- a/bacula/src/stored/vtape_dev.h +++ b/bacula/src/stored/vtape_dev.h @@ -1,7 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2016 Kern Sibbald + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -95,11 +95,12 @@ public: int d_ioctl(int fd, ioctl_req_t request, char *op=NULL); ssize_t d_read(int, void *buffer, size_t count); ssize_t d_write(int, const void *buffer, size_t count); - bool offline(); + bool offline(DCR *dcr); boffset_t lseek(DCR *dcr, off_t offset, int whence) { return -1; } boffset_t lseek(int fd, off_t offset, int whence) { return ::lseek(fd, offset, whence); } + const char *print_type(); }; @@ -107,13 +108,18 @@ public: class vtape: public DEVICE { public: + + vtape(); + ~vtape(); int d_open(const char *pathname, int flags) { return -1; } ssize_t d_read(int fd, void *buffer, size_t count) { return -1; } ssize_t d_write(int fd, const void *buffer, size_t count) { return -1; } int d_close(int) { return -1; } int d_ioctl(int fd, ioctl_req_t request, char *mt=NULL) { return -1; } boffset_t lseek(DCR *dcr, off_t offset, int whence) { return -1; } -}; + bool open_device(DCR *dcr, int omode) { return true; } + const char *print_type(); +}; #endif /* USE_VTAPE */ diff --git a/bacula/src/stored/wait.c b/bacula/src/stored/wait.c index 655e7570d9..3007595499 100644 --- a/bacula/src/stored/wait.c +++ b/bacula/src/stored/wait.c @@ -11,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -23,7 +23,6 @@ * Code for wait_for_sysop() pulled from askdir.c * * Kern Sibbald, March 2005 - * */ @@ -293,23 +292,60 @@ bool wait_for_device(DCR *dcr, int &retries) } -#ifdef xxx /* - * The jcr timers are used for waiting on any device * + * This routine initializes the device wait timers + */ +void init_device_wait_timers(DCR *dcr) +{ + DEVICE *dev = dcr->dev; + JCR *jcr = dcr->jcr; + + /* ******FIXME******* put these on config variables */ + dev->min_wait = 60 * 60; + dev->max_wait = 24 * 60 * 60; + dev->max_num_wait = 9; /* 5 waits =~ 1 day, then 1 day at a time */ + dev->wait_sec = dev->min_wait; + dev->rem_wait_sec = dev->wait_sec; + dev->num_wait = 0; + dev->poll = false; + + jcr->min_wait = 60 * 60; + jcr->max_wait = 24 * 60 * 60; + jcr->max_num_wait = 9; /* 5 waits =~ 1 day, then 1 day at a time */ + jcr->wait_sec = jcr->min_wait; + jcr->rem_wait_sec = jcr->wait_sec; + jcr->num_wait = 0; + +} + +void init_jcr_device_wait_timers(JCR *jcr) +{ + /* ******FIXME******* put these on config variables */ + jcr->min_wait = 60 * 60; + jcr->max_wait = 24 * 60 * 60; + jcr->max_num_wait = 9; /* 5 waits =~ 1 day, then 1 day at a time */ + jcr->wait_sec = jcr->min_wait; + jcr->rem_wait_sec = jcr->wait_sec; + jcr->num_wait = 0; +} + + +/* + * The dev timers are used for waiting on a particular device + * * Returns: true if time doubled * false if max time expired */ -static bool double_jcr_wait_time(JCR *jcr) +bool double_dev_wait_time(DEVICE *dev) { - jcr->wait_sec *= 2; /* double wait time */ - if (jcr->wait_sec > jcr->max_wait) { /* but not longer than maxtime */ - jcr->wait_sec = jcr->max_wait; + dev->wait_sec *= 2; /* double wait time */ + if (dev->wait_sec > dev->max_wait) { /* but not longer than maxtime */ + dev->wait_sec = dev->max_wait; } - jcr->num_wait++; - jcr->rem_wait_sec = jcr->wait_sec; - if (jcr->num_wait >= jcr->max_num_wait) { + dev->num_wait++; + dev->rem_wait_sec = dev->wait_sec; + if (dev->num_wait >= dev->max_num_wait) { return false; } return true; } -#endif diff --git a/bacula/src/streams.h b/bacula/src/streams.h index e1f7dc7236..16a3159945 100644 --- a/bacula/src/streams.h +++ b/bacula/src/streams.h @@ -1,8 +1,7 @@ /* Bacula(R) - The Network Backup Solution - Copyright (C) 2000-2015 Kern Sibbald - Copyright (C) 2000-2014 Free Software Foundation Europe e.V. + Copyright (C) 2000-2017 Kern Sibbald The original author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -12,7 +11,7 @@ Public License, v3.0 ("AGPLv3") and some additional permissions and terms pursuant to its AGPLv3 Section 7. - This notice must be preserved when any source code is + This notice must be preserved when any source code is conveyed and/or propagated. Bacula(R) is a registered trademark of Kern Sibbald. @@ -21,7 +20,6 @@ * Stream definitions. Split from baconfig.h Nov 2010 * * Kern Sibbald, MM - * */ #ifndef __BSTREAMS_H @@ -101,6 +99,9 @@ #define STREAM_ENCRYPTED_FILE_COMPRESSED_DATA 32 /* Encrypted, compressed data */ #define STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA 33 /* Encrypted, compressed Win32 BackupRead data */ +#define STREAM_ADATA_BLOCK_HEADER 200 /* Adata block header */ +#define STREAM_ADATA_RECORD_HEADER 201 /* Adata record header */ + /* * Additional Stream definitions. Once defined these must NEVER * change as they go on the storage media. diff --git a/bacula/src/tools/Makefile.in b/bacula/src/tools/Makefile.in index 136d2468fe..f9f6eb9b66 100644 --- a/bacula/src/tools/Makefile.in +++ b/bacula/src/tools/Makefile.in @@ -68,9 +68,10 @@ bregtest: Makefile bregtest.o ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L../lib -o $@ bregtest.o -lbac -lm $(DLIB) $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) dbcheck: Makefile dbcheck.o ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) \ - ../cats/libbacsql$(DEFAULT_ARCHIVE_TYPE) ../cats/libbaccats$(DEFAULT_ARCHIVE_TYPE) $(DIRCONFOBJS) - $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L../lib -L../cats -o $@ dbcheck.o $(DIRCONFOBJS) \ - -lbaccats -lbacsql -lbaccfg -lbac -lm $(ZLIBS) $(DB_LIBS) $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + ../cats/libbacsql$(DEFAULT_ARCHIVE_TYPE) ../cats/libbaccats$(DEFAULT_ARCHIVE_TYPE) $(DIRCONFOBJS) \ + ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L. -L../lib -L../findlib -L../cats -o $@ dbcheck.o $(DIRCONFOBJS) \ + $(DLIB) -lbaccats -lbacsql -lbacfind -lbaccfg -lbac -lm $(DB_LIBS) $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) fstype: Makefile fstype.o ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -L../lib -L../findlib -o $@ fstype.o -lbacfind -lbac -lm \ @@ -111,9 +112,9 @@ bwild: Makefile ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) bwild.o $(DLIB) -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bbatch: Makefile ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) ../cats/libbacsql$(DEFAULT_ARCHIVE_TYPE) \ - ../cats/libbaccats$(DEFAULT_ARCHIVE_TYPE) bbatch.o - $(LIBTOOL_LINK) $(CXX) -g $(LDFLAGS) -L../cats -L. -L../lib -o $@ bbatch.o \ - -lbaccats -lbacsql -lbac -lm $(ZLIBS) $(DB_LIBS) $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) + ../cats/libbaccats$(DEFAULT_ARCHIVE_TYPE) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) bbatch.o + $(LIBTOOL_LINK) $(CXX) -g $(LDFLAGS) -L../cats -L. -L../lib -L../findlib -o $@ bbatch.o \ + $(DLIB) -lbaccats -lbacsql -lbacfind -lbac -lm $(ZLIBS) $(DB_LIBS) $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bvfs_test: Makefile ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) \ ../cats/libbacsql$(DEFAULT_ARCHIVE_TYPE) ../cats/libbaccats$(DEFAULT_ARCHIVE_TYPE) bvfs_test.o diff --git a/bacula/src/tools/cats_test.c b/bacula/src/tools/cats_test.c index 75adc4d303..f06bba943e 100644 --- a/bacula/src/tools/cats_test.c +++ b/bacula/src/tools/cats_test.c @@ -59,7 +59,7 @@ PROG_COPYRIGHT " -l maximum tuple to fetch\n" " -q print only errors\n" " -v verbose\n" -" -? print this message\n\n"), 2011, BDEMO, VERSION, BDATE); +" -? print this message\n\n"), 2011, "", VERSION, BDATE); exit(1); } diff --git a/bacula/src/tools/dbcheck.c b/bacula/src/tools/dbcheck.c index 0eaacdc734..5c67fd91c1 100644 --- a/bacula/src/tools/dbcheck.c +++ b/bacula/src/tools/dbcheck.c @@ -180,7 +180,7 @@ int main (int argc, char *argv[]) if (argc > 0) { Pmsg0(0, _("Warning skipping the additional parameters for working directory/dbname/user/password/host.\n")); } - config = new_config_parser(); + config = New(CONFIG()); parse_dir_config(config, configfile, M_ERROR_TERM); LockRes(); foreach_res(catalog, R_CATALOG) { diff --git a/bacula/src/tools/testfind.c b/bacula/src/tools/testfind.c index 8b4537ebd0..de3f863d4d 100644 --- a/bacula/src/tools/testfind.c +++ b/bacula/src/tools/testfind.c @@ -127,7 +127,7 @@ main (int argc, char *const *argv) argc -= optind; argv += optind; - config = new_config_parser(); + config = New(CONFIG()); parse_dir_config(config, configfile, M_ERROR_TERM); MSGS *msg; @@ -162,8 +162,7 @@ main (int argc, char *const *argv) free_jcr(jcr); if (config) { - config->free_resources(); - free(config); + delete config; config = NULL; } diff --git a/bacula/src/version.h b/bacula/src/version.h index 4efa6c2a94..2c0a59a59e 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -5,9 +5,9 @@ #define COMMUNITY 1 /* Define to create a Windows community binary */ /* Note: there can be only *one* VERSION in this file */ -#define VERSION "7.4.8" -#define BDATE "10 April 2017" -#define LSMDATE "10Apr17" +#define VERSION "7.9.0" +#define BDATE "08 May 2017" +#define LSMDATE "08May17" #define RELEASE 1 /* Use ONLY in rpms */ @@ -66,8 +66,6 @@ # define SMCHECK #endif -#define BEEF 0 - /* * _USE_LOCKMGR does lock/unlock mutex tracking (dead lock) * it can always be turned on, but we advise to use it only diff --git a/regress/all-changer-tests b/regress/all-changer-tests index 9b6f161ff5..1b24bdc915 100755 --- a/regress/all-changer-tests +++ b/regress/all-changer-tests @@ -1,12 +1,16 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - -# -# ./run all tape tests +# ./run all changer tests # + +if test x$FORCE_CLOUD = xyes ; then + echo "All changer tests skipped for FORCE_CLOUD" + exit 0 +fi + echo "Start autochanger tests" echo "Start autochanger tests" >>test.out rm -f dumps/* diff --git a/regress/all-disk-tests b/regress/all-disk-tests index fd1b94e345..671e0c4bd5 100755 --- a/regress/all-disk-tests +++ b/regress/all-disk-tests @@ -1,11 +1,9 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - -# -# Run all tests +# ./run all tests # echo " " echo " " >>test.out @@ -13,92 +11,124 @@ echo "Start non-root disk tests" echo "Start non-root disk tests" >>test.out rm -f dumps/* ./run tests/acl-xattr-test -./run tests/action-on-purge-test +./run tests/aligned-bug-8013-test ./run tests/allowcompress-test -./run tests/accurate-test +./run tests/auto-label-many-test ./run tests/auto-label-test +./run tests/accurate-test ./run tests/backup-bacula-test -./run tests/bextract-test -./run tests/bconsole-test +./run tests/backup-to-null ./run tests/base-job-test +# +# console-acl-test is broken and has +# orphaned buffers +#./run tests/console-acl-test +# +./run tests/bconsole-test +./run tests/bextract-test ./run tests/big-vol-test +#./run tests/bpipe-test -- errors +./run tests/broken-media-bug-2-test ./run tests/bscan-test +./run tests/btape-test ./run tests/bsr-opt-test +./run tests/console-dotcmd-test +./run tests/cancel-multiple-test ./run tests/comment-test ./run tests/compressed-test -./run tests/lzo-test ./run tests/compress-encrypt-test -./run tests/lzo-encrypt-test ./run tests/concurrent-jobs-test -./run tests/copy-job-test ./run tests/copy-jobspan-test +./run tests/copy-job-test ./run tests/copy-uncopied-test ./run tests/copy-upgrade-test ./run tests/copy-volume-test +./run tests/crazy-smaller-vol-test ./run tests/data-encrypt-test +./run tests/aligned-test ./run tests/delete-test +./run tests/differential-test ./run tests/encrypt-bug-test ./run tests/estimate-test ./run tests/exclude-dir-test ./run tests/fifo-test ./run tests/fileregexp-test -./run tests/backup-to-null -./run tests/regexwhere-test -# The following two can uses *lots* of disk space -# so they are normally turned off, but if you have -# at least two GB free, you can run them -#./run tests/sparse-encrypt-test -#./run tests/gigaslam-sparse-test -./run tests/differential-test ./run tests/four-concurrent-jobs-test ./run tests/four-jobs-test +./run tests/hardlink-test ./run tests/incremental-test +./run tests/jobmedia-bug-test +./run tests/lzo-encrypt-test +./run tests/lzo-test +./run tests/maxbw-test +./run tests/maxbytes-test +./run tests/maxtime-test +./run tests/maxuseduration-test +./run tests/maxvol2-test +./run tests/maxvol-test +./run tests/messages-test +./run tests/messages-saved-test +./run tests/migration-job-purge-test +./run tests/migration-jobspan-test +./run tests/migration-job-test +./run tests/migration-time-test +./run tests/migration-volume-test +./run tests/multi-storage-test +./run tests/next-pool-test +./run tests/next-vol-test +./run tests/next-vol-bug-7302 +./run tests/poll-interval-test +./run tests/pool-attributes-test +./run tests/prune-base-job-test +./run tests/prune-copy-test +./run tests/prune-migration-test +./run tests/prune-pool-test +./run tests/prune-test ./run tests/query-test ./run tests/recycle-test +./run tests/regexwhere-test +./run tests/restart2-base-job-test +./run tests/restart2-job-test +./run tests/restart-accurate-job-test +./run tests/restart-job-test +./run tests/restart-sd-test +./run tests/restart-reschedule-test ./run tests/restore2-by-file-test ./run tests/restore-by-file-test ./run tests/restore-disk-seek-test +./run tests/restore-stop-read-test +./run tests/restore-stop-read2-test +./run tests/restore-multi-session-test +#./run tests/remote-console-test ./run tests/runscript-test -./run tests/source-addr-test -./run tests/stats-test +./run tests/sd-sd-test ./run tests/six-vol-test +./run tests/source-addr-test ./run tests/span-vol-test -./run tests/maxbytes-test -./run tests/maxtime-test -./run tests/maxuseduration-test -./run tests/maxvol-test -./run tests/maxvol2-test -./run tests/messages-test -./run tests/next-vol-test ./run tests/sparse-compressed-test ./run tests/sparse-lzo-test ./run tests/sparse-test +./run tests/stats-test ./run tests/strip-test +./run tests/tls-duplicate-job-test +./run tests/tls-test +./run tests/truncate-test +./run tests/truncate2-test ./run tests/two-jobs-test ./run tests/two-vol-test ./run tests/verify-data-test ./run tests/verify-cat-test ./run tests/verify-vol-test ./run tests/verify-voltocat-test +./run tests/virtual-backup-test +./run tests/virtual-backup2-test +./run tests/virtual-changer-test +./run tests/virtual-jobid-test +./run tests/virtualfull-bug-7154 ./run tests/weird-files2-test ./run tests/weird-files-test -./run tests/migration-job-test -./run tests/migration-job-purge-test -./run tests/migration-jobspan-test -./run tests/migration-volume-test -./run tests/migration-time-test -./run tests/multi-storage-test -# Disabled because certain pruning turned off -#./run tests/prune-test -#./run tests/prune-config-test -./run tests/prune-migration-test -./run tests/prune-copy-test -./run tests/prune-base-job-test -./run tests/hardlink-test -./run tests/tls-duplicate-job-test -./run tests/tls-test -./run tests/virtual-changer-test -./run tests/virtual-backup-test +./run tests/2media-virtual-test +#./run tests/priority-test # broken echo "End non-root disk tests" echo "End non-root disk tests" >>test.out @@ -109,6 +139,7 @@ echo " " >>test.out echo "Start non-root virtual disk autochanger tests" echo "Start non-root virtual disk autochanger tests" >>test.out #./run tests/three-pool-recycle-test +./run tests/three-pool-virtual-test ./run tests/two-pool-test ./run tests/fast-two-pool-test ./run tests/two-volume-test @@ -121,7 +152,5 @@ echo "Start non-root virtual disk autochanger tests" >>test.out ./run tests/three-pool-test ./run tests/2drive-3pool-test ./run tests/2drive-swap-test -# Turn back on after next_vol.c bug is fixed -# ./run tests/next-vol-test echo "End non-root virtual disk autochanger tests" echo "End non-root virtual disk autochanger tests" >>test.out diff --git a/regress/all-tape-tests b/regress/all-tape-tests index d4a3e3bcab..5ecd21bec2 100755 --- a/regress/all-tape-tests +++ b/regress/all-tape-tests @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # ./run all tape tests # @@ -13,12 +11,24 @@ # If we have an autoloader, load the tape in slot1 if test ! x$AUTOCHANGER = x/dev/null ; then a=`bin/mtx-changer $AUTOCHANGER loaded $SLOT1 $TAPE_DRIVE $DRIVE1` - if test $a = 0 ; then + if test $a -eq 0 ; then bin/mtx-changer $AUTOCHANGER load $SLOT1 $TAPE_DRIVE $DRIVE1 fi fi echo " " echo " " >>test.out +if test x$FORCE_DEDUP = xyes ; then + echo "All tape tests skipped for FORCE_DEDUP" + exit 0 +fi +if test x$FORCE_ALIGNED = xyes ; then + echo "All tape tests skipped for FORCE_ALIGNED" + exit 0 +fi +if test x$FORCE_CLOUD = xyes ; then + echo "All tape tests skipped for FORCE_CLOUD" + exit 0 +fi echo "Start all non-root tape tests" echo "Start all non-root tape tests" >>test.out rm -f dumps/* diff --git a/regress/do_all_tests b/regress/do_all_tests index 1c6ae64062..ffbff3c4f8 100755 --- a/regress/do_all_tests +++ b/regress/do_all_tests @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @@ -9,10 +9,20 @@ if [ ! -f bin/tape_options ] ; then touch bin/tape_options fi echo " " >test.out -git status >>test.out 2>&1 -cat build/config.out >>test.out +if [ x$PREBUILT != xyes ]; then + git status >>test.out 2>&1 + cat build/config.out >>test.out +fi echo " " >>test.out -echo "Test results" >>test.out +if [ x$FORCE_DEDUP = xyes ]; then + echo "Test results with dedup" >>test.out +elif [ x$FORCE_ALIGNED = xyes ]; then + echo "Test results with aligned" >>test.out +elif [ x$FORCE_CLOUD = xyes ]; then + echo "Test results with cloud" >>test.out +else + echo "Test results" >>test.out +fi echo " " >>test.out ./starttime ./all-disk-tests diff --git a/regress/scripts/aligned-bacula-dir.conf.in b/regress/scripts/aligned-bacula-dir.conf.in new file mode 100644 index 0000000000..549381526f --- /dev/null +++ b/regress/scripts/aligned-bacula-dir.conf.in @@ -0,0 +1,822 @@ +# +# Default Bacula Director Configuration file +# +# Tweeked for a bigger fileset for Aligned testing +# + +Director { # define myself + Name = @hostname@-dir + DIRPort = @dirport@ # where we listen for UA connections + QueryFile = "@scriptdir@/query.sql" + WorkingDirectory = "@working_dir@" + PidDirectory = "@piddir@" + SubSysDirectory = "@subsysdir@" + PluginDirectory = "@sbindir@" + Maximum Concurrent Jobs = 4 + Heartbeat Interval = 330 + Password = "pNvX1WiXnwv2C/F7E52LGvw6rKjbbPvu2kyuPa9pVaL3" # Console password + Messages = Standard +} + +# +# Define the main nightly save backup job +# By default, this job will back up to disk in @tmpdir@ +Job { + Name = "NightlySave" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 10 + SpoolData=yes + Max Run Time = 30min + Reschedule On Error = no + Reschedule Interval = 10 + Reschedule Times = 5 +} + + +# +# Define the main nightly save backup job +# By default, this job will back up to disk in @tmpdir@ +Job { + Name = "FSType" + Type = Backup + Client=@hostname@-fd + FileSet="FSTypeFS" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 10 + SpoolData=yes +} + +Job { + Name = "Simple" + Type = Backup + Client=@hostname@-fd + FileSet="SimpleSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min +# SpoolData=yes +} + + +Job { + Name = "MonsterSave" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = File1 + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + Maximum Concurrent Jobs = 10 + SpoolData=yes +} + +Job { + Name = "MonsterFileSet" + Type = Backup + Client=@hostname@-fd + FileSet="MonsterFileSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes +} + + + +Job { + Name = "VerifyVolume" + Type = Verify + Level = VolumeToCatalog + Client=@hostname@-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min +} + +Job { + Name = "VerifyData" + Type = Verify + Level = Data + Client=@hostname@-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min +} + + +Job { + Name = "SparseTest" + Type = Backup + Client=@hostname@-fd + FileSet="SparseSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes + Maximum Concurrent Jobs = 10 +} + +Job { + Name = "DeltaTest" + Type = Backup + Client=@hostname@-fd + FileSet="DeltaSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + Maximum Concurrent Jobs = 10 +} + +Job { + Name = "CompressedTest" + Type = Backup + Client=@hostname@-fd + FileSet="CompressedSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes +} + +Job { + Name = "HardlinkTest" + Type = Backup + Client=@hostname@-fd + FileSet="HardlinkSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + Accurate = yes +} + +Job { + Name = "SparseCompressedTest" + Type = Backup + Client=@hostname@-fd + FileSet="SparseCompressedSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes + Maximum Concurrent Jobs = 10 +} + +Job { + Name = "vSphereTest" + Type = Backup + Client=@hostname@-fd + FileSet="vSphereSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min +} + +Job { + Name = "LZOTest" + Type = Backup + Client=@hostname@-fd + FileSet="LZOSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes +} + +Job { + Name = "SparseLZOTest" + Type = Backup + Client=@hostname@-fd + FileSet="SparseLZOSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes + Maximum Concurrent Jobs = 10 +} + +Job { + Name = "FIFOTest" + Type = Backup + Client=@hostname@-fd + FileSet="FIFOSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + ClientRunBeforeJob = "sleep 5" + Max Run Time = 30min +} + + + +# Backup the catalog database (after the nightly save) +Job { + Name = "BackupCatalog" + Type = Backup + Client=@hostname@-fd + FileSet="Catalog" +# Schedule = "WeeklyCycleAfterBackup" + Storage = File + Messages = Standard + Pool = Default + # This creates an ASCII copy of the catalog + RunBeforeJob = "@sbindir@/make_catalog_backup -u regress" + # This deletes the copy of the catalog + RunAfterJob = "@sbindir@/delete_catalog_backup" + Write Bootstrap = "@working_dir@/BackupCatalog.bsr" + Max Run Time = 30min +} + +JobDefs { + Name = "BackupJob" + Type = Backup + Pool = Default + Storage = File + Messages = Standard + Priority = 10 + Max Run Time = 30min +} + +Job { + JobDefs = "BackupJob" + Name = "bug621-job-1" + Client = @hostname@-fd + FileSet="Full Set" + ClientRunBeforeJob = "sleep 5" +} + +Job { + JobDefs = "BackupJob" + Name = "bug621-job-2" + Client = @hostname@-fd + FileSet = "Full Set" + Max Run Time = 30 + Priority = 15 +} + + +# Standard Restore template, to be changed by Console program +Job { + Name = "RestoreFiles" + Type = Restore + Client=@hostname@-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Where = @tmpdir@/bacula-restores + Max Run Time = 30min +} + + +# List of files to be backed up +FileSet { + Name = "Full Set" + Include { + File = /usr/bin + File = <@tmpdir@/file-list + } +} + +# List of files to be backed up +FileSet { + Name = "FSTypeFS" + Include { + Options { + signature=MD5 + fstype = ext4 + } + File = <@tmpdir@/file-list + } +} + + +# List of files to be backed up +FileSet { + Name = "FSTypeFSEmpty" + Include { + Options { + signature=MD5 + fstype = debugfs + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "FSno5" + Include { Options { verify=s5 } + File = <@tmpdir@/file-list + } + Include { Options { sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { compression=GZIP } + File = <@tmpdir@/file-list + } + Include { Options { compression=LZO } + File = <@tmpdir@/file-list + } + Include { Options { compression=LZO; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { compression=GZIP; sparse=yes } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "FS5" + Include { Options { signature=MD5; verify=s5 } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=GZIP } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=LZO } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=LZO; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=GZIP; sparse=yes } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "FS5nocheck" + Include { Options { verify="nog"; signature=MD5 } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=GZIP } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=LZO } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=LZO; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { signature=MD5; compression=GZIP; sparse=yes } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "FS1" + Include { Options { verify=s1; signature=SHA1 } + File = <@tmpdir@/file-list + } + Include { Options { signature=SHA1; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { signature=SHA1; compression=GZIP } + File = <@tmpdir@/file-list + } + Include { Options { signature=SHA1; compression=LZO } + File = <@tmpdir@/file-list + } + Include { Options { signature=SHA1; compression=LZO; sparse=yes } + File = <@tmpdir@/file-list + } + Include { Options { signature=SHA1; compression=GZIP; sparse=yes } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "SimpleSet" + Include { + Options { signature=MD5 } + File = <@tmpdir@/file-list + } +} + + +FileSet { + Name = vSphereSet + Include { + Options { + signature=MD5 + compression = GZIP1 + } + Plugin = "vsphere: host=@VSPHERE_HOST1@" + } +} + +FileSet { + Name = vSphereSet2 + Include { + Options { + signature=MD5 + compression = GZIP1 + } + Plugin = "vsphere: host=@VSPHERE_HOST1@" + Plugin = "vsphere: host=@VSPHERE_HOST2@" + } +} + +FileSet { + Name = "SparseSet" + Include { + Options { + signature=MD5 + sparse=yes + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "CompressedSet" + Include { + Options { + signature=MD5 + compression=GZIP + onefs=no + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "HardlinkSet" + Include { + Options { + signature=MD5 + hardlinks=yes + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "LZOSet" + Include { + Options { + signature=MD5 + compression=LZO + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "DeltaSet" + Include { + Options { + signature=MD5 + plugin = delta + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "FIFOSet" + Include { + Options { + readfifo = yes + signature=MD5 + } + File = <@tmpdir@/file-list + } +} + + +FileSet { + Name = "SparseCompressedSet" + Include { + Options { + signature=MD5 + compression=GZIP + sparse=yes + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "SparseLZOSet" + Include { + Options { + signature=MD5 + compression=LZO + sparse=yes + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "MonsterFileSet" + Include { + Options { + signature = MD5 + noatime = yes + ignore case = yes + Exclude = yes + RegexDir = "Cache" + RegexDir = "Windows Defender" + RegexDir = "Temporary Internet Files" + RegexDir = "bacula" + RegexDir = "Temp" + + RegexDir = "ATI Technologies" + + RegexDir = "wmdownloads" + RegexDir = "My Music" + RegexDir = "iTunes" + RegexDir = "Cookies" + + RegexFile = "desktop.ini" + RegexFile = "thumbs.db" + RegexFile = "acrobat7.exe" + RegexFile = "acr6win.exe" + RegexFile = "AdbeRdr70_enu_full.exe" + RegexFile = "antivirus10_1_5.exe" + #thunderbird lock file + RegexFile = "parent.lock" + + RegexDir = "Retrospect Restore Points" + + #exclude i386 director of windows installer files + WildDir = "[A-Z]:/i386" + + # Exclude Mozilla-based programs' file caches + WildDir = "[A-Z]:/Documents and Settings/*/Application Data/*/Profiles/*/*/ImapMail" + WildDir = "[A-Z]:/Users/*/Application Data/*/Profiles/*/*/ImapMail" + + # Exclude user's registry files - they're always in use anyway. + WildFile = "[A-Z]:/Documents and Settings/*/Local Settings/Application Data/Microsoft/Windows/usrclass.*" + WildFile = "[A-Z]:/Users/*/Local Settings/Application Data/Microsoft/Windows/usrclass.*" + WildFile = "[A-Z]:/Documents and Settings/*/ntuser.*" + WildFile = "[A-Z]:/Users/*/ntuser.*" + + WildDir = "[A-Z]:/Documents and Settings/*/Recent" + WildDir = "[A-Z]:/Users/*/Recent" + + WildDir = "[A-Z]:/Documents and Settings/*/Local Settings/History" + WildDir = "[A-Z]:/Users/*/Local Settings/History" + + # These are always open and unable to be backed up + WildFile = "[A-Z]:/Documents and Settings/All Users/Application Data/Microsoft/Network/Downloader/qmgr[01].dat" + WildFile = "[A-Z]:/Users/All Users/Application Data/Microsoft/Network/Downloader/qmgr[01].dat" + + #Exclude all of Windows... + WildDir = "[A-Z]:/windows" + WildDir = "[A-Z]:/winnt" + WildDir = "[A-Z]:/winxp" + WildDir = "[A-Z]:/win" + + #symantec antivirus app stuff + WildDir = "[A-Z]:/*/Symantec*" + + #system volume information + WildDir = "[A-Z]:/System Volume Information" + + WildFile = "*.tmp" + # ghost image and spanning files + WildFile = "*.gho" + WildFile = "*.ghs" + + # Recycle bins + WildDir = "[A-Z]:/RECYCLER" + WildDir = "[A-Z]:/RECYCLER" + WildDir = "[A-Z]:/RECYCLED" + WildDir = "[A-Z]:/$RECYCLE.BIN" + + # Swap files + WildFile = "[A-Z]:/pagefile.sys" + + # These are programs and are easier to reinstall than restore from + # backup + WildDir = "[A-Z]:/cygwin" + WildDir = "[A-Z]:/Program Files/Adobe/Acrobat 7.0" + WildDir = "[A-Z]:/Program Files/Adobe/Acrobat 8.0" + + WildDir = "[A-Z]:/Program Files/Common Files/Software Center" + WildDir = "[A-Z]:/Software Center" + + WildDir = "[A-Z]:/Program Files/Grisoft" + WildDir = "[A-Z]:/Program Files/Java" + WildDir = "[A-Z]:/Program Files/Java Web Start" + WildDir = "[A-Z]:/Program Files/JavaSoft" + WildDir = "[A-Z]:/Program Files/Microsoft Office" + WildDir = "[A-Z]:/Program Files/Mozilla Firefox" + WildDir = "[A-Z]:/Program Files/Mozilla Thunderbird" + WildDir = "[A-Z]:/Program Files/mozilla.org" + WildDir = "[A-Z]:/Program Files/OpenOffice*" + } + File = <@tmpdir@/file-list + } +} + + +# +# When to do the backups, full backup on first sunday of the month, +# differential (i.e. incremental since full) every other sunday, +# and incremental backups other days +Schedule { + Name = "WeeklyCycle" + Run = Level=Full 1st sun at 1:05 + Run = Level=Differential 2nd-5th sun at 1:05 + Run = Level=Incremental mon-sat at 1:05 +} + +# This schedule does the catalog. It starts after the WeeklyCycle +Schedule { + Name = "WeeklyCycleAfterBackup" + Run = Level=Full sun-sat at 1:10 +} + +# This is the backup of the catalog +FileSet { + Name = "Catalog" + Include { + Options { + signature=MD5 + } + File = /home/kern/bacula/regress/bin/working/bacula.sql + } +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 10 +} + +# Definiton of file storage device +Storage { + Name = File + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = FileStorage + Media Type = File + Maximum Concurrent Jobs = 10 + Heartbeat Interval = 330 +} + +Storage { + Name = File1 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = FileStorage1 + Media Type = File1 + Maximum Concurrent Jobs = 10 + Heartbeat Interval = 330 +} + +Storage { + Name = File2 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = FileStorage2 + Media Type = File + Maximum Concurrent Jobs = 10 + Heartbeat Interval = 330 +} + +# Definition of DLT tape storage device +#Storage { +# Name = DLTDrive +# Address = @hostname@ # N.B. Use a fully qualified name here +# SDPort = @sdport@ +# Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon +# Device = "HP DLT 80" # must be same as Device in Storage daemon +# Media Type = DLT8000 # must be same as MediaType in Storage daemon +#} + +# Definition of DDS tape storage device +#Storage { +# Name = SDT-10000 +# Address = @hostname@ # N.B. Use a fully qualified name here +# SDPort = @sdport@ +# Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon +# Device = SDT-10000 # must be same as Device in Storage daemon +# Media Type = tape # must be same as MediaType in Storage daemon +#} + +# Definition of 8mm tape storage device +#Storage { +# Name = "8mmDrive" +# Address = @hostname@ # N.B. Use a fully qualified name here +# SDPort = @sdport@ +# Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +# Device = "Exabyte 8mm" +# MediaType = "8mm" +#} + + +# Generic catalog service +Catalog { + Name = MyCatalog + @libdbi@ + dbname = @db_name@; user = @db_user@; password = "@db_password@" +} + +# Reasonable message delivery -- send most everything to email address +# and to the console +Messages { + Name = Standard + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression: %t %e of %c %l\" %r" + operatorcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression: Intervention needed for %j\" %r" +# MailOnError = @job_email@ = all +# operator = @job_email@ = mount + console = all, !skipped, !terminate, !restored +# +# WARNING! the following will create a file that you must cycle from +# time to time as it will grow indefinitely. However, it will +# also keep all your messages if the scroll off the console. +# + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + +Messages { + Name = NoEmail + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression: %t %e of %c %l\" %r" + console = all, !skipped, !terminate +# +# WARNING! the following will create a file that you must cycle from +# time to time as it will grow indefinitely. However, it will +# also keep all your messages if the scroll off the console. +# + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + + +# Default pool definition +Pool { + Name = Default + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year +# Label Format = "TEST-${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}:${NumVols}" +# Simple Label Format = "Backup-" +# Maximum Volume Jobs = 1 +# Maximum Volume Bytes = 1500000 +} diff --git a/regress/scripts/aligned-bacula-sd.conf.in b/regress/scripts/aligned-bacula-sd.conf.in new file mode 100644 index 0000000000..f09667221c --- /dev/null +++ b/regress/scripts/aligned-bacula-sd.conf.in @@ -0,0 +1,143 @@ +# +# Default Bacula Storage Daemon Configuration file +# +# For Bacula release 1.33 +# +# You may need to change the name of your tape drive +# on the "Archive Device" directive in the Device +# resource. If you change the Name and/or the +# "Media Type" in the Device resource, please ensure +# that dird.conf has corresponding changes. +# + +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" + Plugin Directory = "@sbindir@/plugins" + Maximum Concurrent Jobs = 100 +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +# +# Devices supported by this Storage daemon +# To connect, the Director's bacula-dir.conf must have the +# same Name and MediaType. +# + +# Optimized for ZFS Deduplication +# Optimized for HPE StorOnce +Device { + Name = FileStorage + Device Type = Aligned +# Maximum Block Size = 128K +# File Alignment = 128K +# Padding Size = 512 + Maximum Block Size = 64K + Minimum Block Size = 0 + File Alignment = 2K + Padding Size = 2K + Media Type = File + Archive Device = @tmpdir@ + LabelMedia = yes; # lets Bacula label unlabelled media + Random Access = Yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; + Maximum Concurrent Jobs = 10 +# Maximum File Size = 10KB +} + +Device { + Name = FileStorage1 + Media Type = File1 + Archive Device = @tmpdir@ + LabelMedia = yes; # lets Bacula label unlabelled media + Random Access = Yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; +} + +Device { + Name = FileStorage2 + Media Type = File + Archive Device = @tmpdir@ + LabelMedia = yes; # lets Bacula label unlabelled media + Random Access = Yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; +} + + + +#Device { +# Name = "HP DLT 80" +# Media Type = DLT8000 +# Archive Device = /dev/nst0 +# AutomaticMount = yes; # when device opened, read it +# AlwaysOpen = yes; +# RemovableMedia = yes; +#} + +#Device { +# Name = SDT-7000 # +# Media Type = DDS-2 +# Archive Device = /dev/nst0 +# AutomaticMount = yes; # when device opened, read it +# AlwaysOpen = yes; +# RemovableMedia = yes; +#} + +#Device { +# Name = Floppy +# Media Type = Floppy +# Archive Device = /mnt/floppy +# RemovableMedia = yes; +# Random Access = Yes; +# AutomaticMount = yes; # when device opened, read it +# AlwaysOpen = no; +#} + +# +# A very old Exabyte with no end of media detection +# +#Device { +# Name = "Exabyte 8mm" +# Media Type = "8mm" +# Archive Device = /dev/nst0 +# Hardware end of medium = No; +# AutomaticMount = yes; # when device opened, read it +# AlwaysOpen = Yes; +# RemovableMedia = yes; +#} + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/ansi-sd-tape.conf.in b/regress/scripts/ansi-sd-tape.conf.in index cf35cef730..5a71116247 100644 --- a/regress/scripts/ansi-sd-tape.conf.in +++ b/regress/scripts/ansi-sd-tape.conf.in @@ -40,6 +40,8 @@ Device { AlwaysOpen = yes; RemovableMedia = yes; @@sbindir@/tape_options + Control Device = /dev/sg1 + Alert Command = "@scriptdir@/tapealert %l" # Maximum File Size = 1000000 # MaximumVolumeSize = 400M Label Type = ansi diff --git a/regress/scripts/bacula-dir-2client.conf.in b/regress/scripts/bacula-dir-2client.conf.in index 9d65392fc9..b601bd3be9 100644 --- a/regress/scripts/bacula-dir-2client.conf.in +++ b/regress/scripts/bacula-dir-2client.conf.in @@ -39,6 +39,7 @@ Job { Maximum Concurrent Jobs = 4 SpoolData = yes # Prefer Mounted Volumes = no + Max Wait Time = 1mins } # @@ -56,6 +57,7 @@ Job { Write Bootstrap = "@working_dir@/NightlySave.bsr" Maximum Concurrent Jobs = 4 SpoolData = yes + Max Wait Time = 1mins # Prefer Mounted Volumes = no } @@ -73,6 +75,7 @@ Job { SpoolData = yes # Prefer Mounted Volumes = no Client Run Before Job = "sleep 5" + Max Wait Time = 1mins } Job { @@ -87,6 +90,7 @@ Job { Maximum Concurrent Jobs = 4 SpoolData = yes # Prefer Mounted Volumes = no + Max Wait Time = 1mins } Job { diff --git a/regress/scripts/bacula-dir-2media-virtual.conf.in b/regress/scripts/bacula-dir-2media-virtual.conf.in new file mode 100644 index 0000000000..c29f652b22 --- /dev/null +++ b/regress/scripts/bacula-dir-2media-virtual.conf.in @@ -0,0 +1,253 @@ +# +# Default Bacula Director Configuration file +# +# Virtual disk changer with two Media Types and two Archive +# devices. +# + +Director { # define myself + Name = @hostname@-dir + DIRPort = @dirport@ # where we listen for UA connections + QueryFile = "@scriptdir@/query.sql" + WorkingDirectory = "@working_dir@" + PidDirectory = "@piddir@" + Maximum Concurrent Jobs = 100 + Password = "pNvX1WiXnwv2C/F7E52LGvw6rKjbbPvu2kyuPa9pVaL3" + Messages = Daemon + FD Connect Timeout = 2min +} + +Job { + Name = "Virtual" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = Virtual + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 100 + Max Run Time = 30min +} + +Job { + Name = "copy-job" + Type = Copy + Client=@hostname@-fd + FileSet="Full Set" + Storage = Virtual + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 100 + Max Run Time = 30min + Selection Type = Job + Selection Pattern = "Virtual" +} + +Job { + Name = "copy-job-next-pool" + Type = Copy + Client=@hostname@-fd + FileSet="Full Set" + Storage = Virtual + Messages = Standard + Pool = Default + Next Pool = Special + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 100 + Max Run Time = 30min + Selection Type = Job + Selection Pattern = "Virtual" +} + +# Standard Restore template, to be changed by Console program +Job { + Name = "RestoreFiles" + Type = Restore + Client = @hostname@-fd + FileSet = "Full Set" + Storage = Virtual + Messages = Standard + Pool = Default + Where = @tmpdir@/bacula-restores + Max Run Time = 30min +} + + +# List of files to be backed up +FileSet { + Name = "Full Set" + Include { Options { signature=MD5 +# Deduplication = yes + } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "Verify Set" + Include { + Options { + signature=MD5 + verify=pins1 + } + File = <@tmpdir@/file-list + } +} + + + +# +# When to do the backups, full backup on first sunday of the month, +# differential (i.e. incremental since full) every other sunday, +# and incremental backups other days +Schedule { + Name = "WeeklyCycle" + Run = Full 1st sun at 1:05 + Run = Differential 2nd-5th sun at 1:05 + Run = Incremental mon-sat at 1:05 +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 100 +} + +Autochanger { + Name = Virtual2 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = Virtual2 + Media Type = disk2 + Maximum Concurrent Jobs = 100 +} + +# Definition of Virtual autochanger +Autochanger { + Name = Virtual + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = Virtual + Media Type = disk + Maximum Concurrent Jobs = 100 + Autochanger = Virtual +} + +Storage { + Name = vDrive-1 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = Virtual + Media Type = disk + Maximum Concurrent Jobs = 100 + Autochanger = Virtual +} + +Storage { + Name = vDrive-3 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = Virtual + Media Type = disk1 + Maximum Concurrent Jobs = 100 + Autochanger = Virtual +} + +# Generic catalog service +Catalog { + Name = MyCatalog + dbname = @db_name@; user = @db_user@; password = "@db_password@" +} + +# Reasonable message delivery -- send most everything to email address +# and to the console +Messages { + Name = Standard + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression Job %j\) %r\" -s \"Regression: %t %e of %c %l\" %r" + operatorcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression Job %j\) %r\" -s \"Regression: Intervention needed for %j\" %r" + console = all, !skipped, !terminate, !restored + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + +# +# Message delivery for daemon messages (no job). +Messages { + Name = Daemon + mailcommand = "@sbindir@/bsmtp -h @smtp_host@ -f \"\(Bacula regression\) %r\" -s \"Regression daemon message\" %r" +# mail = @job_email@ = all, !skipped + console = all, !skipped, !saved + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + + +# Default pool definition +Pool { + Name = Default + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year +# RecyclePool=Scratch # test forward reference + Next Pool = FullCopy +} + +Pool { + Name = Full + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Next Pool = FullCopy +} + +Pool { + Name = FullCopy + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Storage = Virtual +} + +Pool { + Name = Special + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Storage = Virtual2 +} + +#Pool { +# Name = Scratch +# Pool Type = Backup +# Recycle = yes # Bacula can automatically recycle Volumes +# AutoPrune = yes # Prune expired volumes +# Volume Retention = 365d # one year +# Recycle Pool = Scratch # recycle back here +#} + + +#Pool { +# Name = SpecialScratch +# Pool Type = Backup +# Recycle = yes # Bacula can automatically recycle Volumes +# AutoPrune = yes # Prune expired volumes +# Volume Retention = 365d # one year +# Recycle Pool = SpecialScratch # recycle back here +#} diff --git a/regress/scripts/bacula-dir-tape.conf.in b/regress/scripts/bacula-dir-tape.conf.in index b7f68bb6e9..d438b35f4a 100644 --- a/regress/scripts/bacula-dir-tape.conf.in +++ b/regress/scripts/bacula-dir-tape.conf.in @@ -62,8 +62,8 @@ Job { Job { Name = "NightlySave2" Type = Backup - Client=@hostname@-fd - FileSet="Full Set" + Client = @hostname@-fd + FileSet = "Full Set" Storage = tape Messages = Standard Pool = Default @@ -74,6 +74,23 @@ Job { Max Run Time = 30min } +# Backs up to tape2 which is a remote SD on +# another machine +Job { + Name = "NightlySave3" + Type = Backup + Client = @hostname@-fd + FileSet = "Full Set" + Storage = tape2 + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 100 + SpoolData = yes +# Prefer Mounted Volumes = no + Max Run Time = 30min +} + Job { Name = "VerifyTape" Type = Verify @@ -222,6 +239,21 @@ Storage { # Autochanger = yes } +# Definition tape storage device that is on a +# remote machine +Storage { + Name = tape2 + Address = @hostname@ # For same machine shstore simulation +# Address = @remotehostaddr@ # For real two machine setup + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = tape2 # must be same as Device in Storage daemon + Media Type = tape # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 100 +# Autochanger = yes +} + + # Definition of Virtual storage device Storage { Name = Virtual @@ -234,8 +266,63 @@ Storage { Autochanger = yes } +Storage { + Name = vDrive-1 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = vDrive-1 # must be same as Device in Storage daemon + Media Type = Disk # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 100 + Autochanger = Virtual +} + +Storage { + Name = vDrive-2 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = vDrive-2 # must be same as Device in Storage daemon + Media Type = Disk # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 100 + Autochanger = Virtual +} + +Storage { + Name = vDrive-3 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = vDrive-3 # must be same as Device in Storage daemon + Media Type = Disk # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 100 + Autochanger = Virtual +} +# Definition of Virtual storage device +Storage { + Name = VirtualRestore + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = VirtualRestore # must be same as Device in Storage daemon + Media Type = Disk # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 100 + Autochanger = yes +} + +# Definition of Virtual storage device +Storage { + Name = Virtual2 + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = Virtual2 # must be same as Device in Storage daemon + Media Type = Disk2 # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 100 + Autochanger = yes +} # Generic catalog service Catalog { @@ -309,6 +396,66 @@ Pool { Recycle Pool = Scratch # recycle back here } +Pool { + Name = PoolA-MCJ1 + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + ActionOnPurge = Truncate + Volume Retention = 3d + Maximum Volume Bytes = 20M + Maximum Volume Jobs = 1 + LabelFormat = "PoolA-MCJ1" +} + +Pool { + Name = PoolB-MCJ1 + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + ActionOnPurge = Truncate + Volume Retention = 3d + Maximum Volume Bytes = 20M + Maximum Volume Jobs = 1 + LabelFormat = "PoolB-MCJ1" +} + +Pool { + Name = PoolC-MCJ1 + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + ActionOnPurge = Truncate + Volume Retention = 3d + Maximum Volume Bytes = 20M + Maximum Volume Jobs = 1 + LabelFormat = "PoolC-MCJ1" +} + +Pool { + Name = PoolD-MCJ1 + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + ActionOnPurge = Truncate + Volume Retention = 3d + Maximum Volume Bytes = 20M + Maximum Volume Jobs = 1 + LabelFormat = "PoolD-MCJ1" +} + +Pool { + Name = PoolE-MCJ1 + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + ActionOnPurge = Truncate + Volume Retention = 3d + Maximum Volume Bytes = 20M + Maximum Volume Jobs = 1 + LabelFormat = "PoolE-MCJ1" +} + Pool { Name = Special Pool Type = Backup diff --git a/regress/scripts/bacula-dir-vtape.conf.in b/regress/scripts/bacula-dir-vtape.conf.in index a4ebfc1036..ed0d5083bb 100644 --- a/regress/scripts/bacula-dir-vtape.conf.in +++ b/regress/scripts/bacula-dir-vtape.conf.in @@ -188,25 +188,24 @@ Client { } -Storage { +Autochanger { Name = LTO1 Address = @hostname@ SDPort = @sdport@ Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO Media Type = LTO1 - Autochanger = yes Maximum Concurrent Jobs = 50 } -Storage { +Autochanger { Name = LTO3 Address = @hostname@ SDPort = @sdport@ Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO Media Type = LTO3 - Autochanger = yes + SharedStorage = LTO1 Maximum Concurrent Jobs = 50 } @@ -222,7 +221,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO3_0 Media Type = LTO3 - Autochanger = yes + Autochanger = LTO3 Maximum Concurrent Jobs = 5 } @@ -233,7 +232,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO3_1 Media Type = LTO3 - Autochanger = yes + Autochanger = LTO3 Maximum Concurrent Jobs = 5 } @@ -244,7 +243,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO3_2 Media Type = LTO3 - Autochanger = yes + Autochanger = LTO3 Maximum Concurrent Jobs = 5 } @@ -255,7 +254,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO1_3 Media Type = LTO1 - Autochanger = yes + Autochanger = LTO1 Maximum Concurrent Jobs = 5 } @@ -266,7 +265,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO1_4 Media Type = LTO1 - Autochanger = yes + Autochanger = LTO1 Maximum Concurrent Jobs = 5 } @@ -277,7 +276,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO1_5 Media Type = LTO1 - Autochanger = yes + Autochanger = LTO1 Maximum Concurrent Jobs = 5 } @@ -288,7 +287,7 @@ Storage { Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" Device = LTO1-ANSI_6 Media Type = LTO1-ANSI - Autochanger = yes + Autochanger = LTO1 Maximum Concurrent Jobs = 5 } @@ -314,6 +313,8 @@ Catalog { # and to the console Messages { Name = Standard + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression daemon message\" %r" + mail = @job_email@ = all, !skipped console = all, !skipped, !terminate, !restored append = "@working_dir@/log" = all, !skipped catalog = all, !skipped, !saved @@ -323,6 +324,8 @@ Messages { # Message delivery for daemon messages (no job). Messages { Name = Daemon + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression daemon message\" %r" + mail = @job_email@ = all, !skipped console = all, !skipped, !saved append = "@working_dir@/log" = all, !skipped catalog = all, !skipped, !saved diff --git a/regress/scripts/bacula-dir-vtape2.conf.in b/regress/scripts/bacula-dir-vtape2.conf.in new file mode 100644 index 0000000000..06c063df69 --- /dev/null +++ b/regress/scripts/bacula-dir-vtape2.conf.in @@ -0,0 +1,423 @@ +# +# Bacula Director Configuration file +# +# + +Director { # define myself + Name = @hostname@-dir + DIRPort = @dirport@ # where we listen for UA connections + QueryFile = "@scriptdir@/query.sql" + WorkingDirectory = "@working_dir@" + PidDirectory = "@piddir@" + Maximum Concurrent Jobs = 40 + Password = "pNvX1WiXnwv2C/F7E52LGvw6rKjbbPvu2kyuPa9pVaL3" + Messages = Daemon +} + +# +# Define the main nightly save backup job +# By default, this job will back up to disk in @tmpdir@ +Job { + Name = "NightlySave" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = LTO3 + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + SpoolData = yes + Max Wait Time = 1m +# Prefer Mounted Volumes = no +} + +Job { + Name = "NightlySave1" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = LTO1 + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave1.bsr" + Maximum Concurrent Jobs = 10 + SpoolData = yes +# Prefer Mounted Volumes = no + Client Run Before Job = "sleep 2" +} + +Job { + Name = "NightlySave2" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = LTO1 + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave2.bsr" + Maximum Concurrent Jobs = 10 + SpoolData = yes +# Prefer Mounted Volumes = no +} + +Job { + Name = "NightlySave3" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = LTO1 + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave3.bsr" + Maximum Concurrent Jobs = 10 +# Prefer Mounted Volumes = no +} + +Job { + Name = "VerifyTape" + Type = Verify + Level = VolumeToCatalog + Client=@hostname@-fd + FileSet="Verify Set" + Storage = LTO1 + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 4 + SpoolData = yes +# Prefer Mounted Volumes = no +} + +Job { + Name = "copy-job" + Type = Copy + Level = Full + Client=@hostname@-fd + FileSet="Full Set" + Messages = Standard + Storage = LTO3 + Write Bootstrap = "@working_dir@/copyjob.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = Job + Selection Pattern = ".*Save" +# Allow Duplicate Jobs = No +} + +# Standard Restore template, to be changed by Console program +Job { + Name = "RestoreFiles" + Type = Restore + Client=@hostname@-fd + FileSet="Full Set" + Storage = LTO1 + Messages = Standard + Pool = Default + Where = @tmpdir@/bacula-restores +} + + +# List of files to be backed up +FileSet { + Name = "Full Set" + Include { Options { signature=MD5 } + File = <@tmpdir@/file-list + } +} + +FileSet { + Name = "Verify Set" + Include { + Options { + signature=MD5 + verify=pins1 + } + File = <@tmpdir@/file-list + } +} + + +# Client (File Services) to backup +Client { + Name = @hostname@-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 10 +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd2 + Address = @hostname@ + FDPort = 1@fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 10 +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd3 + Address = @hostname@ + FDPort = 2@fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 10 +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd4 + Address = @hostname@ + FDPort = 3@fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 10 +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd5 + Address = @hostname@ + FDPort = 4@fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 10 +} + + +Storage { + Name = LTO1 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO + Media Type = LTO1 + Autochanger = LTO3 + Maximum Concurrent Jobs = 50 +} + +Autochanger { + Name = LTO3 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO + Media Type = LTO3 + Maximum Concurrent Jobs = 50 +} + +################################################################ +# WARNING: Bacula doesn't support this kind of configuration +# This is for testing purpose only !!!!!!!!!!!! +################################################################ + +Storage { + Name = LTO3_0 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO3_0 + Media Type = LTO3 + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +Storage { + Name = LTO3_1 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO3_1 + Media Type = LTO3 + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +Storage { + Name = LTO3_2 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO3_2 + Media Type = LTO3 + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +Storage { + Name = LTO1_3 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO1_3 + Media Type = LTO1 + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +Storage { + Name = LTO1_4 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO1_4 + Media Type = LTO1 + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +Storage { + Name = LTO1_5 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO1_5 + Media Type = LTO1 + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +Storage { + Name = LTO1-ANSI_6 + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = LTO1-ANSI_6 + Media Type = LTO1-ANSI + Autochanger = LTO3 + Maximum Concurrent Jobs = 5 +} + +# Definition of File storage device +Storage { + Name = File + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = FileStorage # must be same as Device in Storage daemon + Media Type = FileMedia # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 10 +} + +# Generic catalog service +Catalog { + Name = MyCatalog + @libdbi@ + dbname = @db_name@; user = @db_user@; password = "@db_password@" +} + +# Reasonable message delivery -- send most everything to email address +# and to the console +Messages { + Name = Standard + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression daemon message\" %r" + mail = @job_email@ = all, !skipped + console = all, !skipped, !terminate, !restored + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped, !saved +} + +# +# Message delivery for daemon messages (no job). +Messages { + Name = Daemon + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression daemon message\" %r" + mail = @job_email@ = all, !skipped + console = all, !skipped, !saved + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped, !saved +} + + +# Default pool definition +Pool { + Name = Default + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Next Pool = MigrationLTO3 +} + +Pool { + Name = Full + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Next Pool = MigrationLTO3 +} + +Pool { + Name = Test + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + MaximumVolumeJobs = 1 + Next Pool = MigrationLTO3 +} + +Pool { + Name = Inc + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Next Pool = MigrationLTO3 +} + +Pool { + Name = Diff + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Next Pool = MigrationLTO3 +} + +Pool { + Name = Scratch + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Recycle Pool = Scratch # recycle back here + Cleaning Prefix = CLN +} + +Pool { + Name = MigrationLTO3 + Pool Type = Backup + Recycle = yes + AutoPrune = yes + Volume Retention = 2d + Storage = LTO3 +} + +Pool { + Name = MigrationLTO1 + Pool Type = Backup + Recycle = yes + AutoPrune = yes + Volume Retention = 2d + Storage = LTO1 +} diff --git a/regress/scripts/bacula-dir.conf.testrunscript.in b/regress/scripts/bacula-dir.conf.testrunscript.in index b23723c1c7..a6c5e40247 100644 --- a/regress/scripts/bacula-dir.conf.testrunscript.in +++ b/regress/scripts/bacula-dir.conf.testrunscript.in @@ -37,6 +37,28 @@ JobDefs { Maximum Concurrent Jobs = 16 } +JobDefs { + Name = "DefaultJobWithRunscript" + Type = Backup + Level = Incremental + Client = @hostname@-fd + FileSet = FS_TESTJOB + Storage = File + Messages = Standard + Pool = Default + Priority = 10 + RunAfterJob = "echo FromJobDefs" + Maximum Concurrent Jobs = 16 +} + +Job { + Name = "RUN_JOBDEFS" + JobDefs = DefaultJobWithRunscript + FileSet = FS_TESTJOB + RunBeforeJob = "/bin/echo RunBeforeJob" + Run After Job = "/bin/echo RunAfterJob" +} + FileSet { Name = FS_TESTJOB Include { diff --git a/regress/scripts/bacula-sd-2d.conf.in b/regress/scripts/bacula-sd-2d.conf.in index a6f23b70c5..5c1063de4e 100644 --- a/regress/scripts/bacula-sd-2d.conf.in +++ b/regress/scripts/bacula-sd-2d.conf.in @@ -68,3 +68,14 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/bacula-sd-2disk-drive.conf.in b/regress/scripts/bacula-sd-2disk-drive.conf.in index fdc3ce523f..1984da8ceb 100644 --- a/regress/scripts/bacula-sd-2disk-drive.conf.in +++ b/regress/scripts/bacula-sd-2disk-drive.conf.in @@ -1,7 +1,7 @@ # # Default Bacula Storage Daemon Configuration file # -# For Bacula release 1.39 +# For Bacula release 8.7.0 # # You may need to change the name of your tape drive # on the "Archive Device" directive in the Device @@ -34,8 +34,8 @@ Director { # Autochanger { Name = tape - Changer Device = @disk_drive@/conf - Changer Command ="@scriptdir@/disk-changer %c %o %S %a %d" + Changer Device = /dev/null + Changer Command ="" Device = Drive-0, Drive-1 } @@ -43,7 +43,7 @@ Device { Name = Drive-0 Device Type = File Media Type = tape - Archive Device = @disk_drive@/drive0 + Archive Device = @tmpdir@ AutomaticMount = yes; # when device opened, read it Autochanger = yes Drive Index = 0 @@ -56,7 +56,7 @@ Device { Name = Drive-1 Device Type = File Media Type = tape - Archive Device = @disk_drive@/drive1 + Archive Device = @tmpdir@ AutomaticMount = yes; # when device opened, read it Autochanger = yes Drive Index = 1 @@ -185,3 +185,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-2disk-virtual.conf.in b/regress/scripts/bacula-sd-2disk-virtual.conf.in new file mode 100644 index 0000000000..cbe679a7d3 --- /dev/null +++ b/regress/scripts/bacula-sd-2disk-virtual.conf.in @@ -0,0 +1,253 @@ +# +# Default Bacula Storage Daemon Configuration file +# +# You may need to change the name of your tape drive +# on the "Archive Device" directive in the Device +# resource. If you change the Name and/or the +# "Media Type" in the Device resource, please ensure +# that dird.conf has corresponding changes. +# + +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" + Maximum Concurrent Jobs = 100 +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + + +Autochanger { + Name = Virtual + Changer Device = /dev/null + Changer Command = /dev/null + Device = vDrive-1 + Device = vDrive-2 + Device = vDrive-3 + Device = vDrive-4 + Device = vDrive-5 + Device = vDrive-6 + Device = vDrive-7 + Device = vDrive-8 + Device = vDrive-R +} + +Device { + Name = vDrive-1 +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-2 +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 2 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-3 +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 3 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-4 + Media Type = Disk +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 4 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-5 + Media Type = Disk +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 5 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-6 + Media Type = Disk +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 6 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-7 + Media Type = Disk +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 7 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-8 + Media Type = Disk +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 7 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-R + Media Type = File +#A Device Type = Aligned +#A Maximum Block Size = 128K +#A Minimum Block Size = 0 +#A File Alignment = 128K +#A Padding Size = 512 + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 9 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 1 + Volume Poll Interval = 15 + ReadOnly = yes + Label Media = yes + Maximum Volume Size = 50M +# Maximum File Size = 1000000 +} + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-2disk.conf.in b/regress/scripts/bacula-sd-2disk.conf.in index 7f8f33e9fb..82b50c724d 100644 --- a/regress/scripts/bacula-sd-2disk.conf.in +++ b/regress/scripts/bacula-sd-2disk.conf.in @@ -33,8 +33,8 @@ Director { # Autochanger { Name = tape - Changer Device = @disk_drive@/conf - Changer Command ="@scriptdir@/disk-changer %c %o %S %a %d" + Changer Device = /dev/null + Changer Command ="" Device = Drive-0, Drive-1 } @@ -42,7 +42,7 @@ Device { Name = Drive-0 # Device Type = File Media Type = tape - Archive Device = @disk_drive@/drive0 + Archive Device = @tmpdir@ AutomaticMount = yes; # when device opened, read it Autochanger = yes Drive Index = 0 @@ -55,7 +55,7 @@ Device { Name = Drive-1 # Device Type = File Media Type = tape - Archive Device = @disk_drive@/drive1 + Archive Device = @tmpdir@ AutomaticMount = yes; # when device opened, read it Autochanger = yes Drive Index = 1 @@ -118,3 +118,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-2media-virtual.conf.in b/regress/scripts/bacula-sd-2media-virtual.conf.in new file mode 100644 index 0000000000..2dbfe813c3 --- /dev/null +++ b/regress/scripts/bacula-sd-2media-virtual.conf.in @@ -0,0 +1,403 @@ +# +# Storage daemon for testing a Virtual Autochanger +# with 2 Media Types and two different Archive Directories +# + +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" + Maximum Concurrent Jobs = 100 +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +Autochanger { + Name = Virtual + Changer Device = /dev/null + Changer Command ="" + Device = vDrive-0, vDrive-1, vDrive-2 + Device = vDrive-3, vDrive-4, vDrive-5, vDrive-6, vDrive-7, vDrive-8, vDrive-9, vDrive-10, vDrive-11, vDrive-12, vDrive-13, vDrive-14, vDrive-15, vDrive-16, vDrive-17, vDrive-18, vDrive-19 +} + +Device { + Name = vDrive-0 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-1 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-2 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 1 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-3 + Device Type = File + Media Type = disk1 # Different Media Type + Archive Device = @tmpdir@/disk1 # Different directory + AutomaticMount = yes; + Autochanger = yes + Drive Index = 2 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-4 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-5 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-6 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-7 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} +Device { + Name = vDrive-8 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-9 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-10 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-11 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-12 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-13 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-14 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-15 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-16 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-17 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-18 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive-19 + Device Type = File + Media Type = disk + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + + +Autochanger { + Name = Virtual2 + Changer Device = /dev/null + Changer Command ="" + Device = vDrive2-1, vDrive2-2, vDrive2-3 +} + +Device { + Name = vDrive2-1 + Device Type = File + Media Type = disk2 + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive2-2 + Device Type = File + Media Type = disk2 + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 1 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + +Device { + Name = vDrive2-3 + Device Type = File + Media Type = disk2 + Archive Device = @tmpdir@/disk + AutomaticMount = yes; + Autochanger = yes + Drive Index = 2 + AlwaysOpen = yes; + RemovableMedia = yes; + Maximum Concurrent Jobs = 3 + Volume Poll Interval = 15 +# Maximum File Size = 1000000 +} + + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-2tape.conf.in b/regress/scripts/bacula-sd-2tape.conf.in index 94f641a40b..895b104f99 100644 --- a/regress/scripts/bacula-sd-2tape.conf.in +++ b/regress/scripts/bacula-sd-2tape.conf.in @@ -59,3 +59,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-4disk.conf.in b/regress/scripts/bacula-sd-4disk.conf.in new file mode 100644 index 0000000000..04855969af --- /dev/null +++ b/regress/scripts/bacula-sd-4disk.conf.in @@ -0,0 +1,184 @@ +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Default Bacula Storage Daemon Configuration file +# +# You may need to change the name of your tape drive +# on the "Archive Device" directive in the Device +# resource. If you change the Name and/or the +# "Media Type" in the Device resource, please ensure +# that dird.conf has corresponding changes. +# + +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +# +# Devices supported by this Storage daemon +# To connect, the Director's bacula-dir.conf must have the +# same Name and MediaType. +# +Autochanger { + Name = tape + Changer Device = /dev/null + Changer Command ="" + Device = Drive-1, Drive-2, Drive-3, Drive-4 +} + +Device { + Name = Drive-1 # + Device Type = File + Media Type = tape + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Drive-2 # + Device Type = File + Media Type = tape + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 1 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Drive-3 # + Device Type = File + Media Type = tape + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 2 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Drive-4 # + Device Type = File + Media Type = tape + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 3 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + + +Autochanger { + Name = Virtual + Changer Device = /dev/null + Changer Command ="" + Device = Virtual-1, Virtual-2, Virtual-3, Virtual-4 +} + +Device { + Name = Virtual-1 + Device Type = File + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Virtual-2 + Device Type = File + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 1 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Virtual-3 + Device Type = File + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 2 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Virtual-4 + Device Type = File + Media Type = Disk + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 3 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = FileStorage + mediatype = FileMedia + Archive Device = @tmpdir@ + LabelMedia = yes; # lets Bacula label unlabeled media + Random Access = Yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; +} + + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/bacula-sd-btape.conf.in b/regress/scripts/bacula-sd-btape.conf.in index 0b05ecc587..98445ed5e1 100644 --- a/regress/scripts/bacula-sd-btape.conf.in +++ b/regress/scripts/bacula-sd-btape.conf.in @@ -66,3 +66,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-migration.conf.in b/regress/scripts/bacula-sd-migration.conf.in index d4b691ee7c..b19876a140 100644 --- a/regress/scripts/bacula-sd-migration.conf.in +++ b/regress/scripts/bacula-sd-migration.conf.in @@ -33,8 +33,8 @@ Director { # Autochanger { Name = DiskChanger - Changer Device = @disk_drive@/conf - Changer Command ="@scriptdir@/disk-changer %c %o %S %a %d" + Changer Device = /dev/null + Changer Command ="" Device = Drive-0 } @@ -42,7 +42,7 @@ Device { Name = Drive-0 # Device Type = File Media Type = DiskChangerMedia - Archive Device = @disk_drive@/drive0 + Archive Device = @tmpdir@ AutomaticMount = yes; # when device opened, read it Autochanger = yes Drive Index = 0 @@ -70,3 +70,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-tape.conf.in b/regress/scripts/bacula-sd-tape.conf.in index 62934b92c2..fce5ca3349 100644 --- a/regress/scripts/bacula-sd-tape.conf.in +++ b/regress/scripts/bacula-sd-tape.conf.in @@ -41,6 +41,8 @@ Device { AlwaysOpen = yes; RemovableMedia = yes; @@sbindir@/tape_options + Control Device = /dev/sg1 + Alert Command = "@scriptdir@/tapealert %l" # Maximum File Size = 1000000 # MaximumVolumeSize = 400M } @@ -54,3 +56,14 @@ Messages { director = @hostname@-dir = all, !terminate append = "@working_dir@/log1.sd" = all } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/bacula-sd-virtual-tape.conf.in b/regress/scripts/bacula-sd-virtual-tape.conf.in new file mode 100644 index 0000000000..0ebede7f65 --- /dev/null +++ b/regress/scripts/bacula-sd-virtual-tape.conf.in @@ -0,0 +1,80 @@ +# +# Default Bacula Storage Daemon Configuration file +# +# You may need to change the name of your tape drive +# on the "Archive Device" directive in the Device +# resource. If you change the Name and/or the +# "Media Type" in the Device resource, please ensure +# that dird.conf has corresponding changes. +# + +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" + Maximum Concurrent Jobs = 100 +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +Autochanger { + Name = tape + Changer Device = /dev/null + Changer Command = "" + Device = Drive-0, Drive-1 +} + +Device { + Name = Drive-0 + Device Type = File + Media Type = tape + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +Device { + Name = Drive-1 + Device Type = File + Media Type = tape + Archive Device = @tmpdir@ + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 1 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-virtual.conf.in b/regress/scripts/bacula-sd-virtual.conf.in index 22b44d158d..25f09dbd21 100644 --- a/regress/scripts/bacula-sd-virtual.conf.in +++ b/regress/scripts/bacula-sd-virtual.conf.in @@ -84,3 +84,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/bacula-sd-vtape.conf.in b/regress/scripts/bacula-sd-vtape.conf.in index 8b9e3a4345..9e6cb79635 100644 --- a/regress/scripts/bacula-sd-vtape.conf.in +++ b/regress/scripts/bacula-sd-vtape.conf.in @@ -184,3 +184,15 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/broken-media-bug-2-bacula-sd.conf.in b/regress/scripts/broken-media-bug-2-bacula-sd.conf.in index a2c61323c5..3fc0eb9d8f 100644 --- a/regress/scripts/broken-media-bug-2-bacula-sd.conf.in +++ b/regress/scripts/broken-media-bug-2-bacula-sd.conf.in @@ -105,3 +105,14 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/cleanup b/regress/scripts/cleanup index e7a7add119..9f5fbbb53f 100755 --- a/regress/scripts/cleanup +++ b/regress/scripts/cleanup @@ -1,13 +1,14 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2016 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # -# # Cleanup left over files -- both before and after test run # . scripts/functions + cwd=`pwd` +chmod -R u+rwx ${tmp} rm -rf ${tmp}/FileVolume* rm -rf ${tmp}/TestVolume001 ${tmp}bacula-restores ${tmp}/Small* rm -rf ${tmp}/sed_tmp ${tmp}/file-list @@ -15,10 +16,12 @@ rm -rf ${tmp}/* rm -rf ${working}/log ${tmp}/TEST-* rm -rf ${working}/log ${tmp}/Backup-* rm -rf ${working}/*.bsr ${working}/log*.sd -rm -rf ${working}/*.trace ${working}/*.traceback ${working}/*.lockdump +rm -rf ${working}/*.trace ${working}/*.traceback ${working}/*.state rm -rf ${working}/@* +rm -rf ${working}/* rm -rf ${tmp}disk-changer rm -f ${cwd}/bin/plugins/test-plugin-fd.so +rm -rf ${cwd}/mnt find . -name "gigaslam.gif" -exec rm -f {} \; @@ -27,13 +30,40 @@ if [ -d ${bin} ] ; then if [ -f ${bin}/bacula ] ; then ${bin}/bacula stop -KILL 2>&1 >/dev/null fi - cd ${scripts} - ./drop_bacula_tables >/dev/null 2>&1 - ./make_bacula_tables >/dev/null 2>&1 - ./grant_bacula_privileges >/dev/null 2>&1 - cd ${cwd} fi +cd ${scripts} +./drop_bacula_tables >/dev/null 2>&1 +./make_bacula_tables >/dev/null 2>&1 +./grant_bacula_privileges >/dev/null 2>&1 +cd ${cwd} + if [ x$USE_VTAPE = xyes ]; then rm -rf ${working}/ach fi + +if [ x$FORCE_DEDUP = xyes ]; then + rm -rf ${working}/dde + rm -rf ${working}/ddefd +fi + +# Remove cloud test volumes +if [ x$FORCE_CLOUD = xyes ]; then + drv=`echo $CLOUD_DRIVER | tr 'A-Z' 'a-z'` + if [ "${drv}" = s3 -o "${drv}" = fakes3 ]; then + if [ x${CLOUD_HOSTNAME} != x ]; then + if [ x${CLOUD_PROTOCOL} = xHTTPS ]; then + endpoint="" + else + endpoint="--endpoint-url=http://${CLOUD_HOSTNAME}" + fi + fi + if [ x${CLOUD_REGION} != x ]; then + region="--region ${CLOUD_REGION}" + else + region="" + fi + aws configure set default.s3.signature_version s3v4 + aws s3 rm s3://${CLOUD_BUCKETNAME} $endpoint --recursive --include 'TestVol*' --include 'Vol*' $region 2>/dev/null >/dev/null + fi +fi diff --git a/regress/scripts/copy-2client-confs b/regress/scripts/copy-2client-confs index 30e2a16662..af169a549d 100755 --- a/regress/scripts/copy-2client-confs +++ b/regress/scripts/copy-2client-confs @@ -1,15 +1,14 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - cp -f scripts/bacula-dir-2client.conf ${conf}/bacula-dir.conf cp -f scripts/bacula-sd-2disk-drive.conf ${conf}/bacula-sd.conf cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf cp -f scripts/test-console.conf ${conf}/bconsole.conf cp -f scripts/test-console.conf ${conf}/bat.conf -outf="tmp/sed_tmp" +outf="tmp/sed_tmp2" echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} cp ${conf}/bacula-dir.conf tmp/1 sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-2disk-confs b/regress/scripts/copy-2disk-confs index ab2e60dca3..6b3d63aabe 100755 --- a/regress/scripts/copy-2disk-confs +++ b/regress/scripts/copy-2disk-confs @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Setup for using the Virtual Disk Changer (simulates tape changer) # @@ -12,7 +10,7 @@ cp -f scripts/bacula-sd-2disk.conf ${conf}/bacula-sd.conf cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf cp -f scripts/test-console.conf ${conf}/bconsole.conf cp -f scripts/test-console.conf ${conf}/bat.conf -outf="tmp/sed_tmp" +outf="tmp/sed_tmp2" echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} cp ${conf}/bacula-dir.conf tmp/1 sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-2disk-drive-confs b/regress/scripts/copy-2disk-drive-confs index 8c2f59bdf1..0d318c3b03 100755 --- a/regress/scripts/copy-2disk-drive-confs +++ b/regress/scripts/copy-2disk-drive-confs @@ -1,15 +1,14 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - cp -f scripts/bacula-dir-tape.conf ${conf}/bacula-dir.conf cp -f scripts/bacula-sd-2disk-drive.conf ${conf}/bacula-sd.conf cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf cp -f scripts/test-console.conf ${conf}/bconsole.conf cp -f scripts/test-console.conf ${conf}/bat.conf -outf="tmp/sed_tmp" +outf="tmp/sed_tmp2" echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} cp ${conf}/bacula-dir.conf tmp/1 sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-2disk-tape-confs b/regress/scripts/copy-2disk-tape-confs new file mode 100755 index 0000000000..54ac1688bf --- /dev/null +++ b/regress/scripts/copy-2disk-tape-confs @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +cp -f scripts/bacula-dir-tape.conf ${conf}/bacula-dir.conf +cp -f scripts/bacula-sd-virtual-tape.conf ${conf}/bacula-sd.conf +cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf +cp -f scripts/test-console.conf ${conf}/bconsole.conf +cp -f scripts/test-console.conf ${conf}/bat.conf +outf="tmp/sed_tmp2" +echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} +cp ${conf}/bacula-dir.conf tmp/1 +sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-2disk-virtual-confs b/regress/scripts/copy-2disk-virtual-confs new file mode 100755 index 0000000000..25fa071e94 --- /dev/null +++ b/regress/scripts/copy-2disk-virtual-confs @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +cp -f scripts/bacula-dir-tape.conf ${conf}/bacula-dir.conf +cp -f scripts/bacula-sd-2disk-virtual.conf ${conf}/bacula-sd.conf +cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf +cp -f scripts/test-console.conf ${conf}/bconsole.conf +cp -f scripts/test-console.conf ${conf}/bat.conf +outf="tmp/sed_tmp2" +echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} +cp ${conf}/bacula-dir.conf tmp/1 +sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-2drive-confs b/regress/scripts/copy-2drive-confs index 1f7db20c97..2272d12b4e 100755 --- a/regress/scripts/copy-2drive-confs +++ b/regress/scripts/copy-2drive-confs @@ -1,15 +1,14 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - cp -f scripts/bacula-dir-tape.conf ${conf}/bacula-dir.conf cp -f scripts/bacula-sd-2drive.conf ${conf}/bacula-sd.conf cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf cp -f scripts/test-console.conf ${conf}/bconsole.conf cp -f scripts/test-console.conf ${conf}/bat.conf -outf="tmp/sed_tmp" +outf="tmp/sed_tmp2" echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} cp ${conf}/bacula-dir.conf tmp/1 sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-2tape-confs b/regress/scripts/copy-2tape-confs index 689d664154..9f3ee29e51 100755 --- a/regress/scripts/copy-2tape-confs +++ b/regress/scripts/copy-2tape-confs @@ -1,15 +1,14 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - cp -f scripts/bacula-dir-tape.conf ${conf}/bacula-dir.conf cp -f scripts/bacula-sd-2tape.conf ${conf}/bacula-sd.conf cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf cp -f scripts/test-console.conf ${conf}/bconsole.conf cp -f scripts/test-console.conf ${conf}/bat.conf -outf="tmp/sed_tmp" +outf="tmp/sed_tmp2" echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} cp ${conf}/bacula-dir.conf tmp/1 sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-4disk-confs b/regress/scripts/copy-4disk-confs new file mode 100755 index 0000000000..6e0c464f15 --- /dev/null +++ b/regress/scripts/copy-4disk-confs @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# +# Setup for using the Virtual Disk Changer (simulates tape changer) +# +cp -f scripts/bacula-dir-tape.conf ${conf}/bacula-dir.conf +cp -f scripts/bacula-sd-4disk.conf ${conf}/bacula-sd.conf +cp -f scripts/test-bacula-fd.conf ${conf}/bacula-fd.conf +cp -f scripts/test-console.conf ${conf}/bconsole.conf +cp -f scripts/test-console.conf ${conf}/bat.conf +outf="tmp/sed_tmp2" +echo "s%# Autochanger = yes% Autochanger = yes%g" >${outf} +cp ${conf}/bacula-dir.conf tmp/1 +sed -f ${outf} tmp/1 >${conf}/bacula-dir.conf diff --git a/regress/scripts/copy-btape-confs b/regress/scripts/copy-btape-confs index 699ef339c9..619ced8ea4 100755 --- a/regress/scripts/copy-btape-confs +++ b/regress/scripts/copy-btape-confs @@ -1,9 +1,8 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - /bin/cp -f ${rscripts}/bacula-dir-tape.conf ${conf}/bacula-dir.conf /bin/cp -f ${rscripts}/bacula-sd-btape.conf ${conf}/bacula-sd.conf /bin/cp -f ${rscripts}/test-bacula-fd.conf ${conf}/bacula-fd.conf diff --git a/regress/scripts/copy-tls-crypto-confs b/regress/scripts/copy-tls-crypto-confs new file mode 100755 index 0000000000..a88bf62a56 --- /dev/null +++ b/regress/scripts/copy-tls-crypto-confs @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +cp -f scripts/tls-bacula-dir.conf ${conf}/bacula-dir.conf +cp -f scripts/tls-bacula-sd.conf ${conf}/bacula-sd.conf +cp -f scripts/tls-crypto-bacula-fd.conf ${conf}/bacula-fd.conf +cp -f scripts/test-console.conf ${conf}/bconsole.conf +cp -f scripts/test-console.conf ${conf}/bat.conf +cp -f scripts/tls-CA.pem ${conf}/tls-CA.pem +cp -f scripts/tls-cert.pem ${conf}/tls-cert.pem +cp -f scripts/cryptokeypair.pem ${conf}/cryptokeypair.pem +cp -f scripts/master2048.cert ${conf}/master2048.cert diff --git a/regress/scripts/do_sed b/regress/scripts/do_sed index 049624d533..c9b7c34ca7 100755 --- a/regress/scripts/do_sed +++ b/regress/scripts/do_sed @@ -1,18 +1,24 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # -# cwd=`pwd` . scripts/functions mkdir -p ${bin} +mkdir -p ${tmp} out="${tmp}/sed_tmp" ${rscripts}/create_sed # process .in files with sed script +#sed -f ${out} ${rscripts}/bacula-dir.conf.in >${rscripts}/bacula-dir.conf +#sed -f ${out} ${rscripts}/bacula-fd.conf.in >${rscripts}/bacula-fd.conf +#sed -f ${out} ${rscripts}/bacula-sd.conf.in >${rscripts}/bacula-sd.conf +#sed -f ${out} ${rscripts}/bconsole.conf.in >${rscripts}/bconsole.conf +sed -f ${out} ${rscripts}/bacula-sd-2disk-virtual.conf.in >${rscripts}/bacula-sd-2disk-virtual.conf +sed -f ${out} ${rscripts}/bacula-sd-virtual-tape.conf.in >${rscripts}/bacula-sd-virtual-tape.conf sed -f ${out} ${rscripts}/bacula-dir.conf.errors.in >${rscripts}/bacula-dir.conf.errors sed -f ${out} ${rscripts}/bacula-dir.conf.accurate.in >${rscripts}/bacula-dir.conf.accurate sed -f ${out} ${rscripts}/test-bacula-dir.conf.in >${rscripts}/test-bacula-dir.conf @@ -25,11 +31,16 @@ sed -f ${out} ${rscripts}/testa-bacula-dir.conf.in >${rscripts}/testa-bacula-dir sed -f ${out} ${rscripts}/testb-bacula-dir.conf.in >${rscripts}/testb-bacula-dir.conf sed -f ${out} ${rscripts}/test-bacula-fd.conf.in >${rscripts}/test-bacula-fd.conf sed -f ${out} ${rscripts}/test-bacula-sd.conf.in >${rscripts}/test-bacula-sd.conf +sed -f ${out} ${rscripts}/aligned-bacula-sd.conf.in >${rscripts}/aligned-bacula-sd.conf +sed -f ${out} ${rscripts}/aligned-bacula-dir.conf.in >${rscripts}/aligned-bacula-dir.conf +sed -f ${out} ${rscripts}/migrate-bacula-dir.conf.in >${rscripts}/migrate-bacula-dir.conf +sed -f ${out} ${rscripts}/migrate-bacula-sd.conf.in >${rscripts}/migrate-bacula-sd.conf sed -f ${out} ${rscripts}/virtualfull-extreme-bacula-dir.conf.in >${rscripts}/virtualfull-extreme-bacula-dir.conf sed -f ${out} ${rscripts}/test-console.conf.in >${rscripts}/test-console.conf sed -f ${out} ${rscripts}/crypto-bacula-fd.conf.in >${rscripts}/crypto-bacula-fd.conf sed -f ${out} ${rscripts}/bacula-dir-tape.conf.in >${rscripts}/bacula-dir-tape.conf sed -f ${out} ${rscripts}/bacula-dir-2client.conf.in >${rscripts}/bacula-dir-2client.conf +sed -f ${out} ${rscripts}/bacula-dir-2media-virtual.conf.in >${rscripts}/bacula-dir-2media-virtual.conf sed -f ${out} ${rscripts}/bacula-dir-fifo.conf.in >${rscripts}/bacula-dir-fifo.conf sed -f ${out} ${rscripts}/bacula-dir-strip.conf.in >${rscripts}/bacula-dir-strip.conf sed -f ${out} ${rscripts}/bacula-dir-migration.conf.in >${rscripts}/bacula-dir-migration.conf @@ -42,8 +53,11 @@ sed -f ${out} ${rscripts}/bacula-sd-2tape.conf.in >${rscripts}/bacula-sd-2tape.c sed -f ${out} ${rscripts}/bacula-sd-migration.conf.in >${rscripts}/bacula-sd-migration.conf sed -f ${out} ${rscripts}/bacula-sd-virtual.conf.in >${rscripts}/bacula-sd-virtual.conf sed -f ${out} ${rscripts}/bacula-sd-2disk.conf.in >${rscripts}/bacula-sd-2disk.conf +sed -f ${out} ${rscripts}/bacula-sd-4disk.conf.in >${rscripts}/bacula-sd-4disk.conf +sed -f ${out} ${rscripts}/bacula-sd-2media-virtual.conf.in >${rscripts}/bacula-sd-2media-virtual.conf sed -f ${out} ${rscripts}/bacula-sd-2drive.conf.in >${rscripts}/bacula-sd-2drive.conf sed -f ${out} ${rscripts}/bacula-sd-2disk-drive.conf.in >${rscripts}/bacula-sd-2disk-drive.conf +sed -f ${out} ${rscripts}/bacula-sd-2disk-virtual.conf.in >${rscripts}/bacula-sd-2disk-virtual.conf sed -f ${out} ${rscripts}/broken-media-bug-bacula-dir.conf.in >${rscripts}/broken-media-bug-bacula-dir.conf sed -f ${out} ${rscripts}/broken-media-bug-2-bacula-dir.conf.in >${rscripts}/broken-media-bug-2-bacula-dir.conf sed -f ${out} ${rscripts}/broken-media-bug-2-bacula-sd.conf.in >${rscripts}/broken-media-bug-2-bacula-sd.conf @@ -53,9 +67,11 @@ sed -f ${out} ${rscripts}/cleanup-2drive.in >${rscripts}/cleanup-2drive sed -f ${out} ${rscripts}/prepare-two-tapes.in >${rscripts}/prepare-two-tapes sed -f ${out} ${rscripts}/bacula-dir.conf.testrunscript.in >${rscripts}/bacula-dir.conf.testrunscript sed -f ${out} ${rscripts}/bacula-dir-vtape.conf.in >${rscripts}/bacula-dir-vtape.conf +sed -f ${out} ${rscripts}/bacula-dir-vtape2.conf.in >${rscripts}/bacula-dir-vtape2.conf sed -f ${out} ${rscripts}/bacula-sd-vtape.conf.in >${rscripts}/bacula-sd-vtape.conf sed -f ${out} ${rscripts}/prepare-fake-autochanger.in >${rscripts}/prepare-fake-autochanger sed -f ${out} ${rscripts}/tls-bacula-fd.conf.in >${rscripts}/tls-bacula-fd.conf +sed -f ${out} ${rscripts}/tls-crypto-bacula-fd.conf.in >${rscripts}/tls-crypto-bacula-fd.conf sed -f ${out} ${rscripts}/tls-bacula-sd.conf.in >${rscripts}/tls-bacula-sd.conf sed -f ${out} ${rscripts}/tls-bacula-dir.conf.in >${rscripts}/tls-bacula-dir.conf diff --git a/regress/scripts/functions b/regress/scripts/functions index 9a21653ee4..2722e42d91 100644 --- a/regress/scripts/functions +++ b/regress/scripts/functions @@ -1,7 +1,11 @@ # +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# # A set of useful functions to be sourced in each test # +. ./config check_encoding() { @@ -16,24 +20,90 @@ start_test() check_encoding rm -rf ${working}/@* variant_name="" - + devicetype=0 # If no or dummy smtp_host, use dummy email handler if [ x${SMTP_HOST} = x -o x${SMTP_HOST} = xdummy ]; then cp scripts/dummy_bsmtp bin/bsmtp chmod 755 bin/bsmtp else # Deactivate all email - outf="${tmp}/sed_tmp" + outf="tmp/sed_tmp2" echo "s% mail =%# mail = %g" >${outf} echo "s% operator =%# operator =%g" >>${outf} cp ${conf}/bacula-dir.conf ${tmp}/1 sed -f ${outf} ${tmp}/1 > ${conf}/bacula-dir.conf fi + echo $TestName | grep aligned > /dev/null + if [ $? -eq 0 -o x$FORCE_ALIGNED = xyes ]; then + make -C build/src/stored install-aligned > /dev/null 2>&1 + $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Device Type", "Aligned", "Device")' + $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Plugin Directory", "$plugins", "Storage")' + variant_name="Aligned" + devicetype=`expr $devicetype + 1` + fi + if [ x$FORCE_CLOUD = xyes ]; then + make -C build/src/stored install-cloud > /dev/null 2>&1 + grep DummyCloud $conf/bacula-sd.conf > /dev/null + if [ $? != 0 ]; then + $bperl -e 'extract_resource("$rscripts/test-bacula-sd.conf", "Cloud", "DummyCloud")' >> $conf/bacula-sd.conf + fi + $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Device Type", "Cloud", "Device")' + $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Cloud", "DummyCloud", "Device")' + $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Plugin Directory", "$plugins", "Storage")' + mkdir -p $tmp/cloud + if [ "$CLOUD_HOSTNAME" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'HostName', '\"$CLOUD_HOSTNAME\"', 'Cloud')" + fi + if [ "$CLOUD_BUCKETNAME" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'BucketName', '\"$CLOUD_BUCKETNAME\"', 'Cloud')" + fi + if [ "$CLOUD_ACCESSKEY" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'AccessKey', '\"$CLOUD_ACCESSKEY\"', 'Cloud')" + fi + if [ "$CLOUD_SECRETKEY" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'SecretKey', '\"$CLOUD_SECRETKEY\"', 'Cloud')" + fi + if [ "$CLOUD_REGION" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'Region', '\"$CLOUD_REGION\"', 'Cloud')" + fi + if [ "$CLOUD_PROTOCOL" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'Protocol', '$CLOUD_PROTOCOL', 'Cloud')" + fi + if [ "$CLOUD_URISTYLE" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'UriStyle', '$CLOUD_URISTYLE', 'Cloud')" + fi + if [ "$CLOUD_DRIVER" ]; then + driver=`echo $CLOUD_DRIVER | tr 'A-Z' 'a-z'` + # FakeS3 is useful but keeps big parts in memory. So we must limit parts. + if [ "$driver" = "fakes3" ]; then + if [ "$CLOUD_MAXIMUM_PART_SIZE" = "" ]; then + CLOUD_MAXIMUM_PART_SIZE=50MB + fi + CLOUD_DRIVER=S3 + fi + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'Driver', '\"$CLOUD_DRIVER\"', 'Cloud')" + fi + if [ "$CLOUD_MAXIMUM_PART_SIZE" ]; then + $bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumPartSize', '$CLOUD_MAXIMUM_PART_SIZE', 'Device')" + fi + + variant_name="Cloud" + devicetype=`expr $devicetype + 1` + fi + if [ x$FORCE_SDCALLS = xyes ]; then + $bperl -e 'add_attribute("$conf/bacula-dir.conf", "SD Calls Client", "yes", "Client")' + variant_name="${variant_name} SDcall" + fi + if [ $devicetype -gt 1 ]; then + echo "ERROR: Found multiple FORCE_XXX switches" + set | awk '/^FORCE_/ { print $0 }' + exit 1 + fi ./test_starttime echo " " echo " " - echo " === Starting $TestName ${variant_name} at `date +%R:%S` ===" - echo " === Starting $TestName ${variant_name} at `date +%R:%S` ===" >> ${working}/log + echo " === Start $TestName ${variant_name} at `date +%R:%S` ===" + echo " === Start $TestName ${variant_name} at `date +%R:%S` ===" >> ${working}/log echo " " export TestName export zstat @@ -42,9 +112,32 @@ start_test() zstat=0 bstat=0 rstat=0 + vstat=0 dstat=0 } +# Change setup so that we run with shared storage plugin +setup_shstore() +{ +if test x${USE_SHSTORE} = xyes ; then + require_tape_drive + /bin/cp -f ${rscripts}/bacula-sd-lock-changer.conf ${scripts}/bacula-sd.conf + scripts/cleanup-tape + + # install the shstore plugin + rm -f bin/plugins/shstore-sd.so + make -C build/src/plugins/sd install-shstore-plugin >/dev/null +fi +} + +require_cloud() +{ + if [ "$FORCE_CLOUD" != yes ]; then + echo "$TestName can run only with FORCE_CLOUD" + exit 0 + fi +} + require_root() { MUID=`/usr/bin/id | awk -F= '{print $2}' | awk -F\( '{print $1}'` @@ -60,6 +153,16 @@ fi require_tape_drive() { +# tape not comptible with aligned +if test x$FORCE_ALIGNED = xyes; then + echo "$TestName test not compatible with FORCE_ALIGNED." + exit 0 +fi +# tape not comptible with cloud +if test x$FORCE_CLOUD = xyes; then + echo "$TestName test not compatible with FORCE_CLOUD." + exit 0 +fi if test x${TAPE_DRIVE} = x/dev/null ; then echo "$TestName test needs a tape drive, but has none." exit 1 @@ -84,17 +187,54 @@ fi require_vtape() { +# vtape not comptible with aligned +if test x$FORCE_ALIGNED = xyes; then + echo "$TestName test not compatible with FORCE_ALIGNED." + exit 0 +fi +# vtape not comptible with cloud +if test x$FORCE_CLOUD = xyes; then + echo "$TestName test not compatible with FORCE_CLOUD." + exit 0 +fi if test x${USE_VTAPE} = x ; then echo "$TestName test needs the vtape driver." exit 0 fi } +require_disk() +{ +# vtape not comptible with aligned +if test x$FORCE_ALIGNED = xyes; then + echo "$TestName test not compatible with FORCE_ALIGNED." + exit 0 +fi +# vtape not comptible with cloud +if test x$FORCE_CLOUD = xyes; then + echo "$TestName test not compatible with FORCE_CLOUD." + exit 0 +fi +if test x${USE_VTAPE} = xyes ; then + echo "$TestName test not compatible with FORCE_VTAPE." + exit 0 +fi +} + +require_changer() +{ +# disk changer not comptible with cloud +if test x$FORCE_CLOUD = xyes; then + echo "$TestName test not compatible with FORCE_CLOUD." + exit 0 +fi +} + require_linux() { os=`uname` if [ $os != 'Linux' ]; then - echo "This test $TestName runs only on Linux" + echo "$TestName test runs only on Linux" exit 0 fi } @@ -175,6 +315,7 @@ run_bacula() zstat=0 estat=0 if test "$debug" -eq 1 ; then + ${scripts}/bacula-ctl-sd start -m ${scripts}/bacula-ctl-fd start -m $1 ${scripts}/bacula-ctl-dir start -m @@ -233,11 +374,44 @@ bscan_libdbi() stop_bacula() { + if [ "$CHECK_JOBMEDIA" != 0 ]; then + $bperl -e 'check_jobmedia()' + if [ $? -ne 0 ]; then + echo " " + echo " !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! " + echo " JobMedia is corrupted" + echo " " >>test.out + echo " " >>test.out + echo " !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! " >>test.out + echo " JobMedia is corrupted" >>test.out + echo " " + exit 1 + fi + fi + if [ "$FORCE_CLOUD" = yes ]; then + $bperl -e 'check_parts()' + if [ $? -ne 0 ]; then + estat=1 + fi + fi if test "$debug" -eq 1 ; then ${scripts}/bacula stop else ${scripts}/bacula stop 2>&1 >/dev/null fi + # Any File/Dir with @ as the first character is a lock file + ls ${working}/@* 2>/dev/null 1>/dev/null + if test $? -eq 0 ; then + echo " " + echo " !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! " + echo " SCSI lock still set" + echo " " >>test.out + echo " " >>test.out + echo " !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! " >>test.out + echo " SCSI lock still set" >>test.out + echo " " + exit 1 + fi } check_for_zombie_jobs() @@ -263,8 +437,11 @@ change_jobname() check_two_logs() { + bstat=${bstat:-99} # We must find at least one job in log1.out grep "^ Termination: *Backup OK" ${tmp}/log1.out 2>&1 >/dev/null - bstat=${bstat:-$?} + if test $? -ne 0; then + bstat=2 + fi grep "^ Termination: .*Backup Error" ${tmp}/log1.out 2>&1 >/dev/null if test $? -eq 0; then bstat=2 @@ -289,7 +466,7 @@ check_two_logs() fi grep "^ Termination: .*Verify Differences" ${tmp}/log2.out 2>&1 >/dev/null if test $? -eq 0; then - rstat=4 + vstat=4 fi grep "Encoding error for database" ${tmp}/log1.out > /dev/null if test $? -eq 0; then @@ -301,6 +478,28 @@ check_two_logs() print_debug "Found orphaned buffers" estat=1 fi + if [ x$REGRESS_CHECK_CORRUPTION = xyes ]; then + perl -Mscripts::functions -e "check_volumes('$tmp/log1.out', '$tmp/log2.out')" 2>&1 >/dev/null + if test $? -ne 0; then + print_debug "Found volume corruption" + estat=1 + fi + fi +} + +die_test() +{ + code=$1 + msg=$2 + + print_debug $msg + stop_bacula + + if test "$code" -gt "$estat" ; then + estat=$code + fi + end_test + exit 1 } dtitle() @@ -405,14 +604,13 @@ check_restore_tmp_build_diff() # estat is special error status (shown by print_debug message) # rstat is restore status # zstat is zombie job(s) +# vstat is verify status # end_test() { if [ x$notracedump != xyes ]; then cat ${working}/bacula.*.traceback 2>/dev/null cp -f ${working}/bacula.*.traceback ${dumps} 2>/dev/null - cat ${working}/*.lockdump 2>/dev/null - cp -f ${working}/*.lockdump ${dumps} 2>/dev/null fi if [ -f $tmp/err.log ]; then cat $tmp/err.log @@ -421,48 +619,49 @@ end_test() t=`date +%R:%S` if [ $estat != 0 ] ; then echo " " - echo " !!!!! $TestName failed!!! $t $d !!!!! " - echo " Status: estat=$estat zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" + echo " !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " + echo " Status: estat=$estat zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" echo " " >>test.out - echo " !!!!! $TestName failed!!! $t $d !!!!! " >>test.out - echo " Status: estat=$estat zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" >>test.out + echo " !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " >>test.out + echo " Status: estat=$estat zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out echo " " exit 1 fi if [ $zstat != 0 ] ; then echo " " - echo " !!!!! $TestName failed!!! $t $d !!!!! " - echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" + echo " !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " + echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" echo " " >>test.out - echo " !!!!! $TestName failed!!! $t $d !!!!! " >>test.out - echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" >>test.out + echo " !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " >>test.out + echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out echo " " exit 1 fi - if [ $dstat != 0 -o $bstat != 0 -o $rstat != 0 ] ; then + if [ $dstat != 0 -o $bstat != 0 -o $rstat != 0 -o $vstat != 0 ] ; then echo " " - echo " !!!!! $TestName failed!!! $t $d !!!!! " - echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" + echo " !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " + echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" echo " " >>test.out - echo " !!!!! $TestName failed!!! $t $d !!!!! " >>test.out - echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" >>test.out - if [ $bstat != 0 -o $rstat != 0 ] ; then + echo " !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " >>test.out + echo " Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out + if [ $bstat != 0 -o $rstat != 0 -o $vstat != 0 ] ; then echo " !!! Bad termination status !!! " echo " !!! Bad termination status !!! " >>test.out else echo " !!! Restored files differ !!! " echo " !!! Restored files differ !!! " >>test.out fi - echo " Status: backup=$bstat restore=$rstat diff=$dstat" - echo " Status: backup=$bstat restore=$rstat diff=$dstat" >>test.out + echo " Status: backup=$bstat restore=$rstat diff=$dstat verify=$vstat" + echo " Status: backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out echo " Test owner of $SITE_NAME is $EMAIL" echo " Test owner of $SITE_NAME is $EMAIL" >>test.out echo " " >>test.out echo " " exit 1 else - echo " ===== $TestName OK $t $d ===== " - echo " ===== $TestName OK $t $d ===== " >>test.out + # KES -- remove variant to make line fit in 80 chars + echo " ===== End $TestName OK $t $d ===== " + echo " ===== End $TestName OK $t $d ===== " >>test.out if test "$debug" -eq 0 ; then ${rscripts}/cleanup fi @@ -489,6 +688,21 @@ disable_plugins() done } +update_win32() +{ + if [ -d $cwd/build/src/win32/release32 \ + -o -d $cwd/build/src/win32/release64 ] \ + || [ -d $cwd/release32 -o -d $cwd/release64 ] \ + || [ -d $cwd/../bacula/src/win32/release32 \ + -o -d $cwd/../bacula/src/win32/release64 ] + then + echo "Try to upgrade the FileDaemon:\t" + wget -qO - "$WIN32_ADDR:8091/install" + else + echo "Windows binaries not found, skiping upgrade" + fi +} + debug_wait() { if test "x${REGRESS_WAIT}" = "x1"; then diff --git a/regress/scripts/functions.pm b/regress/scripts/functions.pm index b73277637f..4e1ed546da 100644 --- a/regress/scripts/functions.pm +++ b/regress/scripts/functions.pm @@ -3,19 +3,22 @@ use strict; =head1 LICENSE - Bacula® - The Network Backup Solution + Bacula(R) - The Network Backup Solution - Copyright (C) 2008-2014 Bacula Systems SA + Copyright (C) 2000-2017 Kern Sibbald - The main author of Bacula is Kern Sibbald, with contributions from - many others, a complete list can be found in the file AUTHORS. + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. - Licensees holding a valid Bacula Systems SA license may use this file - and others of this release in accordance with the proprietary license - agreement provided in the LICENSE file. Redistribution of any part of - this release is not permitted. + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. - Bacula® is a registered trademark of Kern Sibbald. + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. =cut @@ -30,23 +33,25 @@ our @EXPORT = qw(update_some_files create_many_files check_multiple_copies update_client $HOST $BASEPORT add_to_backup_list run_bconsole run_bacula start_test end_test create_bconcmds create_many_dirs cleanup start_bacula - get_dirname + get_dirname check_jobmedia_content stop_bacula get_resource set_maximum_concurrent_jobs get_time add_attribute check_prune_list check_min_volume_size init_delta update_delta check_max_backup_size comment_out - create_many_files_size $plugins debug p + create_many_files_size check_jobmedia $plugins debug p check_max_volume_size $estat $bstat $rstat $zstat $cwd $bin $scripts $conf $rscripts $tmp $working $dstat extract_resource $db_name $db_user $db_password $src $tmpsrc $out $CLIENT docmd set_global_maximum_concurrent_jobs check_volumes update_some_files_rep remote_init remote_config remote_stop remote_diff remote_check - get_field_size get_field_ratio create_binfile ); + get_field_size get_field_ratio create_binfile get_bytes get_mbytes + check_parts); use File::Copy qw/copy/; our ($cwd, $bin, $scripts, $conf, $rscripts, $tmp, $working, $estat, $dstat, - $plugins, $bstat, $zstat, $rstat, $debug, $out, $TestName, + $plugins, $bstat, $zstat, $rstat, $debug, $out, $TestName, $FORCE_ALIGNED, + $PREBUILT, $FORCE_CLOUD, $REMOTE_CLIENT, $REMOTE_ADDR, $REMOTE_FILE, $REMOTE_PORT, $REMOTE_PASSWORD, $REMOTE_STORE_ADDR, $REGRESS_DEBUG, $REMOTE_USER, $start_time, $end_time, $db_name, $db_user, $db_password, $src, $tmpsrc, $HOST, $BASEPORT, $CLIENT); @@ -101,6 +106,9 @@ BEGIN { $ENV{REMOTE_PASSWORD} = $REMOTE_PASSWORD = $ENV{REMOTE_PASSWORD} || "xxx"; $ENV{REMOTE_STORE_ADDR}=$REMOTE_STORE_ADDR=$ENV{REMOTE_STORE_ADDR} || undef; $ENV{REMOTE_USER} = $REMOTE_USER = $ENV{REMOTE_USER} || undef; + $ENV{FORCE_ALIGNED} = $FORCE_ALIGNED = $ENV{FORCE_ALIGNED} || 'no'; + $ENV{FORCE_CLOUD} = $FORCE_CLOUD = $ENV{FORCE_CLOUD} || 'no'; + $ENV{PREBUILT} = $PREBUILT = $ENV{PREBUILT} || 'no'; $ENV{CLIENT} = $CLIENT = $ENV{CLIENT} || "$HOST-fd"; $ENV{LANG} = 'C'; $out = ($debug) ? '@tee' : '@out'; @@ -140,6 +148,17 @@ sub cleanup sub start_test { + if ($FORCE_ALIGNED eq "yes") { + if ($PREBUILT ne "yes") { + system("make -C $cwd/build/src/plugins/sd install-aligned-plugin > /dev/null"); + } + add_attribute("$conf/bacula-sd.conf", "Device Type", "Aligned", "Device"); + add_attribute("$conf/bacula-sd.conf", "Plugin Directory", "$plugins", "Storage"); + } + if ($FORCE_CLOUD eq "yes") { + add_attribute("$conf/bacula-sd.conf", "Device Type", "Cloud", "Device"); + } + $start_time = time(); my $d = strftime('%R:%S', localtime($start_time)); print "\n\n === Starting $TestName at $d ===\n"; @@ -344,12 +363,13 @@ sub check_volumes my @files = @_; my %done; unlink("$tmp/check_volumes.out"); + unlink("$tmp/check_volumes_data.out"); foreach my $f (@files) { open(FP, $f) or next; while (my $f = ) { - if ($f =~ /Wrote label to prelabeled Volume "(.+?)" on file device "(.+?)" \((.+?)\)/) { + if ($f =~ /Wrote label to prelabeled Volume "(.+?)" on (?:dedup data|file) device "(.+?)" \((.+?)\)/) { if (!$done{$1}) { $done{$1} = 1; if (-f "$3/$1") { @@ -358,6 +378,11 @@ sub check_volumes debug("Found problems for $1, traces are in $tmp/check_volumes.out"); $estat = 1; } + system("$bin/bextract -t -c $conf/bacula-sd.conf -V \"$1\" \"$2\" /tmp &>> $tmp/check_volumes_data.out"); + if ($? != 0) { + debug("Found problems for $1, traces are in $tmp/check_volumes_data.out"); + $estat = 1; + } } } } @@ -367,6 +392,122 @@ sub check_volumes return $estat; } +# Here we want to list all cloud parts and check what we have in the catalog +sub check_parts +{ + my $tempfile = "$tmp/check_parts.$$"; + open(FP, "|$bin/bconsole -c $conf/bconsole.conf >$tempfile"); + print FP "\@echo File generated by scripts::function::check_part()\n"; + print FP "sql\n"; + print FP "SELECT 'Name', VolumeName, Storage.Name FROM Media JOIN Storage USING (StorageId) WHERE VolType = 14;\n"; + close(FP); + + unlink("$tmp/check_parts.out"); + open(CMD, ">$tmp/bconsole.cmd"); + print CMD "\@output $tmp/check_parts.out\n"; + open(FP, $tempfile); + while (my $l = ) { + $l =~ s/,//g; # Default bacula output is putting , every 1000 + $l =~ s/\|/!/g; # | is a special char in regexp + if ($l =~ /!\s*Name\s*!\s*([\w\d-]+)\s*!\s*([\w\d-]+)\s*/) { + print CMD "cloud list volume=$1 storage=$2\n"; + } + } + close(FP); + close(CMD); + run_bconsole("$tmp/bconsole.cmd"); + open(OUT, "$tmp/check_parts.out"); + while (my $l = ) { + if ($l =~ /Error/) { + print $l; + $estat=1; + } + } + close(OUT); +} + +# This test is supposed to detect JobMedia corruption for all jobs +# stored in the catalog. +sub check_jobmedia +{ + use bigint; + + my %jobids; + my $ret=0; + my %jobs; + # SELECT JobId, Min(FirstIndex) AS A FROM JobMedia GROUP BY JobId HAVING Min(FirstIndex) > 1; + open(FP, "|$bin/bconsole -c $conf/bconsole.conf >$tmp/check_jobmedia.$$"); + print FP "\@echo File generated by scripts::function::check_jobmedia()\n"; + print FP "sql\n"; + print FP "SELECT 'ERROR with FirstIndex not starting at 1 (JobId|FirstIndex)', JobId, Min(FirstIndex) AS A FROM JobMedia GROUP BY JobId HAVING Min(FirstIndex) > 1;\n"; + print FP "SELECT 'ERROR with LastIndex != JobFiles (JobId|LastIndex|JobFiles)', JobId, Max(LastIndex), JobFiles FROM Job JOIN JobMedia USING (JobId) WHERE JobStatus = 'T' AND Type = 'B' GROUP BY JobId,JobFiles HAVING Max(LastIndex) != JobFiles;\n"; + print FP "SELECT 'Index', JobId, FirstIndex, LastIndex, JobMediaId FROM JobMedia ORDER BY JobId, JobMediaId;\n"; + print FP "SELECT 'Block', JobId, MediaId, StartFile, EndFile, StartBlock, EndBlock, JobMediaId FROM JobMedia ORDER BY JobId, JobMediaId;\n"; + print FP "SELECT 'ERROR StartAddress > EndAddress (JobMediaId)', JobMediaId from JobMedia where ((CAST(StartFile AS bigint)<<32) + StartBlock) > ((CAST (EndFile AS bigint) <<32) + EndBlock);\n"; + close(FP); + + my $tempfile = "$tmp/check_jobmedia.$$"; + open(FP, $tempfile); + while (my $l = ) { + $l =~ s/,//g; # Default bacula output is putting , every 1000 + $l =~ s/\|/!/g; # | is a special char in regexp + + if ($l =~ /ERROR with LastIndex [\D]+(\d+)/) { + print $l; + print "HINT: Some FileIndex are not covered by a JobMedia. It usually means that you ", + "can't restore jobs impacted (jobid $1)\n\n"; + $jobids{$1}=1; + $ret++; + + } elsif ($l =~ / ERROR /) { + print $l; + $ret++; + # JobId FirstIndex LastIndex + # Index ! 1 ! 1 ! 2277 ! + } elsif ($l =~ /Index\s*!\s*(\d+)\s*!\s*(\d+)\s*!\s*(\d+)\s*!/) { + my ($jobid, $first, $last) = ($1, $2, $3); + + next if ($first == 0 && $last == 0); + + if ($jobs{$jobid} && !($jobs{$jobid} == $first || $jobs{$jobid} == ($first - 1))) { + print "ERROR: found a gap in JobMedia, the FirstIndex is not equal to the previous LastIndex for jobid $jobid FirstIndex $first LastIndex $last PreviousLast $jobs{$jobid}\n"; + $ret++; + } + $jobs{$jobid} = $last; + + # JobId MediaId StartFile EndFile StartBlock EndBlock JobMediaId + # Block ! 2 ! 3 ! 1 ! 1 ! 129223 ! 999807168 ! 4 ! + } elsif ($l =~ /Block\s*!\s*(\d+)\s*!\s*(\d+)\s*!\s*(\d+)\s*!\s*(\d+)\s*!\s*(\d+)\s*!\s*(\d+)\s*!/) { + my ($jobid, $mediaid, $firstfile, $lastfile, $firstblk, $lastblk) = ($1, $2, $3, $4, $5, $6); + + my $first = ($firstfile << 32) + $firstblk; + my $last = ($lastfile << 32) + $lastblk; + + if ($jobs{"$jobid:$mediaid"} && $jobs{"$jobid:$mediaid"} > $first) { + print "ERROR: in JobMedia, previous Block is before the current Block for jobid=$jobid mediaid=$mediaid ("; + print $jobs{"$jobid:$mediaid"}, " > $first)\n"; + $ret++; + } + if ($last < $first) { + print "ERROR: in JobMedia, the EndAddress is lower than the FirstAddress for JobId=$jobid MediaId=$mediaid ($last < $first)\n"; + $ret++; + } + $jobs{"$jobid:$mediaid"} = $last; + } + } + close(FP); + if ($ret) { + print "ERROR: Found errors while checking JobMedia records, look the file $tempfile\n"; + if (scalar(%jobids)) { + print " The JobId list to check is dumped to $tmp/bad-jobid.out\n"; + open(FP, ">$tmp/bad-jobid.out"); + print FP join("\n", keys %jobids), "\n"; + close(FP); + } + } + exit $ret; +} + # check if a volume is too big # check_max_backup_size(10000, "vol1", "vol3"); sub check_max_volume_size @@ -539,8 +680,13 @@ sub create_many_files for(my $i=0; $i<=$nb; $i++) { $base = chr($i % 26 + 65); open(FP, ">$dest/$base/a${base}a${i}aaa$base") or die "$dest/$base $!"; + print FP "$i\n"; if ($sparse_size) { - seek(FP, $sparse_size + $i, 0); + seek(FP, ($sparse_size + $i)/2, 1); + } + print FP "$i\n"; + if ($sparse_size) { + seek(FP, ($sparse_size + $i)/2, 1); } print FP "$i\n"; close(FP); @@ -1071,6 +1217,49 @@ sub remote_init system("ssh $REMOTE_USER$REMOTE_ADDR 'cd $REMOTE_FILE && perl -Mscripts::functions -e remote_check'"); } +sub get_mbytes +{ + my ($source, $cmd, $binonly) = @_; + my $buf; + if (!open(FP1, $cmd)) { + print "ERR\nCan't open $cmd $@\n"; + exit 1; + } + if (!open(FP, $source)) { + print "ERR\nCan't open $source $@\n"; + exit 1; + } + while (my $l = ) { + if ($l =~ /^(\d+):(\d+)/) { + if (!$binonly) { + print "New chunk is $1:$2\n"; + } + seek(FP, $1, 0); + sysread(FP, $buf, $2); + print $buf; + if (!$binonly) { + print "\n"; + } + } + } + close(FP); + close(FP1); +} + +sub get_bytes +{ + my ($file, $offset, $len) = @_; + my $buf; + if (!open(FP, $file)) { + print "ERR\nCan't open $file $@\n"; + exit 1; + } + seek(FP, $offset, 0); + sysread(FP, $buf, $len); + print $buf, "\n"; + close(FP); +} + sub create_binfile { my ($file, $nb) = @_; @@ -1152,4 +1341,74 @@ sub update_delta return "OK\n"; } +sub check_jobmedia_content +{ + use bigint; + my ($jobmedia, $bls) = @_; + my @lst; + my $jm; + + open(FP, $jobmedia); + +# jobmediaid: 110 +# jobid: 10 +# mediaid: 2 +# volumename: Vol-0002 +# firstindex: 1 +# lastindex: 1 +# startfile: 0 +# endfile: 0 +# startblock: 903,387 +# endblock: 5,096,666 + + while (my $line = ) { + if ($line =~ /(\w+): (.+)/) { + my ($k, $t) = (lc($1), $2); + $t =~ s/,//g; + $jm->{$k} = $t; + + if ($k eq 'endblock') { + $jm->{startaddress} = ($jm->{startfile} << 32) + $jm->{startblock}; + $jm->{endaddress} = ($jm->{endfile} << 32) + $jm->{endblock}; + push @lst, $jm; + $jm = {}; + } + } + } + close(FP); + + open(FP, $bls); + #File:blk=0:11160794 blk_num=0 blen=64512 First rec FI=SOS_LABEL SessId=10 SessTim=1424160078 Strm=10 rlen=152 + my $volume; + while (my $line = ) { + chomp($line); + if ($line =~ /Ready to read from volume "(.+?)"/) { + $volume = $1; + } + if ($line =~ /File:blk=(\d+):(\d+) blk_num=\d+ blen=(\d+)/) { + my $found = 0; + my ($address, $len) = (($1<<32) + $2, $3); + foreach $jm (@lst) { + if ($volume eq $jm->{volumename} && $address >= $jm->{startaddress} && $address <= $jm->{endaddress}) + { + $found = 1; + last; + } + } + if (!$found) { + print "ERROR: Address=$address len=$len volume=$volume not in BSR!!\n"; + print "$line\nJobMedia:\n"; + foreach $jm (@lst) { + if ($volume eq $jm->{volumename}) + { + print "JobMediaId=$jm->{jobmediaid}\tStartAddress=$jm->{startaddress}\tEndAddress=$jm->{endaddress}\n"; + } + } + } + } + } + + close(FP); +} + 1; diff --git a/regress/scripts/migrate-bacula-dir.conf.in b/regress/scripts/migrate-bacula-dir.conf.in new file mode 100644 index 0000000000..bc69a532ee --- /dev/null +++ b/regress/scripts/migrate-bacula-dir.conf.in @@ -0,0 +1,755 @@ +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test for finding Migration errors. Essential +# parts supplied by Arno +# + +Director { # define myself + Name = @hostname@-dir + DIRPort = @dirport@ # where we listen for UA connections + QueryFile = "@scriptdir@/query.sql" + WorkingDirectory = "@working_dir@" + PidDirectory = "@piddir@" + PluginDirectory = "@sbindir@" + Maximum Concurrent Jobs = 51 + Heartbeat Interval = 330 + Password = "pNvX1WiXnwv2C/F7E52LGvw6rKjbbPvu2kyuPa9pVaL3" # Console password + Messages = Standard +} + +File Set { + Name = "IncrData" + Include { + Options { + Signature = SHA1 + Accurate = pnugsiamc + Verify = pnugsiamc1 + No atime = Yes + ACL Support = Yes + Compression = LZO + } + File = <@tmpdir@/file-list + } +} + +Client { + Name = Fake + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" + Maximum Concurrent Jobs = 20 +} + +Job { + Name = "HeiseMig" + Type = Migrate + Client = Fake + File Set = "IncrData" + Messages = "Standard" + Priority = 10 + Maximum Concurrent Jobs = 2 + Enabled = No + Pool = "Heise1" + Selection Type = Job + Selection Pattern = ".*" +} + +# Automated configuration for Heise-test Virtual Full Backups +Job { + Name = "HeiseV01" + Type = Backup + Level = Incremental + Client = "HeiseV01-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV01-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV02" + Type = Backup + Level = Incremental + Client = "HeiseV02-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV02-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV03" + Type = Backup + Level = Incremental + Client = "HeiseV03-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV03-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV04" + Type = Backup + Level = Incremental + Client = "HeiseV04-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV04-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV05" + Type = Backup + Level = Incremental + Client = "HeiseV05-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV05-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV06" + Type = Backup + Level = Incremental + Client = "HeiseV06-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV06-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV07" + Type = Backup + Level = Incremental + Client = "HeiseV07-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV07-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV08" + Type = Backup + Level = Incremental + Client = "HeiseV08-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV08-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV09" + Type = Backup + Level = Incremental + Client = "HeiseV09-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV09-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV10" + Type = Backup + Level = Incremental + Client = "HeiseV10-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV10-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV11" + Type = Backup + Level = Incremental + Client = "HeiseV11-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV11-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV12" + Type = Backup + Level = Incremental + Client = "HeiseV12-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV12-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV13" + Type = Backup + Level = Incremental + Client = "HeiseV13-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV13-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV14" + Type = Backup + Level = Incremental + Client = "HeiseV14-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV14-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV15" + Type = Backup + Level = Incremental + Client = "HeiseV15-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV15-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV16" + Type = Backup + Level = Incremental + Client = "HeiseV16-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV16-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV17" + Type = Backup + Level = Incremental + Client = "HeiseV17-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV17-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV18" + Type = Backup + Level = Incremental + Client = "HeiseV18-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV18-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV19" + Type = Backup + Level = Incremental + Client = "HeiseV19-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV19-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Job { + Name = "HeiseV20" + Type = Backup + Level = Incremental + Client = "HeiseV20-fd" + File Set = "IncrData" + Accurate = Yes + Storage = File + Pool = "Heise1" + Messages = "Standard" + Priority = 10 + Write Bootstrap = "@working_dir@/%n.bsr" + Enabled = No + Maximum Concurrent Jobs = 1 +} + +Client { + Name = HeiseV20-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 6 months + Job Retention = 6 months # six months + AutoPrune = yes # Prune expired Jobs/Files +} + + +Storage { + Name = File + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = VirtCh + Media Type = VirtMedia1 + Autochanger = Yes + Maximum Concurrent Jobs = 10 + Maximum Concurrent Read Jobs = 3 +} + +Storage { + Name = VirtA + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = VirtCh + Media Type = VirtMedia0 + Autochanger = Yes + Maximum Concurrent Jobs = 10 + # 9 drives so we fix read to less than half + Maximum Concurrent Read Jobs = 4 +} + +Storage { + Name = VirtB + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" + Device = VirtCh + Media Type = VirtMedia1 + Autochanger = Yes + Maximum Concurrent Jobs = 50 + Maximum Concurrent Read Jobs = 4 +} + + +Pool { + Name = "Heise1" + Pool Type = Backup + Storage = "VirtB" + Job Retention = 2 hours + File Retention = 100 min + Volume Retention = 3 hours + Maximum Volume Jobs = 7 + Label Format = "Heise1-" + Next Pool = "Heise2" +} + +Pool { + Name = "Heise2" + Pool Type = Backup + Storage = "VirtB" + Job Retention = 2 hours + File Retention = 100 min + Volume Retention = 3 hours + Maximum Volume Jobs = 7 + Label Format = "Heise2-" + Next Pool = "Heise3" +} + +Pool { + Name = "Heise3" + Pool Type = Backup + Storage = "VirtA" + Job Retention = 2 hours + File Retention = 105 min + Volume Retention = 2.5 hours + Maximum Volume Jobs = 1 + Label Format = "Heise3-" +} + + +# Standard Restore template, to be changed by Console program +Job { + Name = "RestoreFiles" + Type = Restore + Client = Fake + FileSet = "Full Set" + Storage = File + Messages = Standard + Pool = Default + Where = @tmpdir@/bacula-restores + Max Run Time = 30min +} + + +# List of files to be backed up +FileSet { + Name = "Full Set" + Include { + Options { signature=MD5; + verify=pins5 } + File = <@tmpdir@/file-list + } +} + + +# Generic catalog service +Catalog { + Name = MyCatalog + dbname = @db_name@; user = @db_user@; password = "@db_password@" +} + +# Reasonable message delivery -- send most everything to email address +# and to the console +Messages { + Name = Standard + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression: %t %e of %c %l\" %r" + operatorcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bacula regression\) %r\" -s \"Regression: Intervention needed for %j\" %r" +# MailOnError = @job_email@ = all +# operator = @job_email@ = mount + console = all, !skipped, !terminate, !restored +# +# WARNING! the following will create a file that you must cycle from +# time to time as it will grow indefinitely. However, it will +# also keep all your messages if the scroll off the console. +# + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + +# Default pool definition +Pool { + Name = Default + Pool Type = Backup + Recycle = yes # Bacula can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year +# Label Format = "TEST-${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}:${NumVols}" +# Simple Label Format = "Backup-" +# Maximum Volume Jobs = 1 +# Maximum Volume Bytes = 1500000 +} diff --git a/regress/scripts/migrate-bacula-sd.conf.in b/regress/scripts/migrate-bacula-sd.conf.in new file mode 100644 index 0000000000..cf0d829458 --- /dev/null +++ b/regress/scripts/migrate-bacula-sd.conf.in @@ -0,0 +1,187 @@ +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# For testing migration +# +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Plugin Directory = "@sbindir@/plugins" + Maximum Concurrent Jobs = 100 +} + +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +Autochanger { + Name = VirtCh + Device = Virt0, Virt1, Virt2, Virt3, Virt4, Virt5, Virt6, Virt7, Virt8 + Changer Command = "" + Changer Device = /dev/null +} + +Device { + Name = Virt0 + Device Type = File + Media Type = VirtMedia0 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 0 + Maximum Concurrent Jobs = 3 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt1 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 1 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt2 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 2 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt3 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 3 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt4 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 4 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt5 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 5 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt6 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 6 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt7 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 7 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +Device { + Name = Virt8 + Device Type = File + Media Type = VirtMedia1 + Archive Device = @tmpdir@ + Automatic Mount = Yes + Always Open = Yes + Removable Media = Yes + Autochanger = Yes + Drive Index = 8 + Maximum Concurrent Jobs = 1 + Label Media = Yes + Maximum File Size = 128k + Maximum Volume Size = 512m +} + +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/new-test-bacula-dir.conf.in b/regress/scripts/new-test-bacula-dir.conf.in index c389f9f0a0..104f9d553f 100644 --- a/regress/scripts/new-test-bacula-dir.conf.in +++ b/regress/scripts/new-test-bacula-dir.conf.in @@ -46,6 +46,23 @@ Job { Reschedule Times = 5 } + +# +# Define the main nightly save backup job +# By default, this job will back up to disk in @tmpdir@ +Job { + Name = "FSType" + Type = Backup + Client=@hostname@-fd + FileSet="FSTypeFS" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 10 + SpoolData=yes +} + Job { Name = "Simple" Type = Backup @@ -159,6 +176,20 @@ Job { SpoolData=yes } +Job { + Name = "HardlinkTest" + Type = Backup + Client=@hostname@-fd + FileSet="HardlinkSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + Accurate = yes +} + Job { Name = "SparseCompressedTest" Type = Backup @@ -299,6 +330,31 @@ FileSet { } } +# List of files to be backed up +FileSet { + Name = "FSTypeFS" + Include { + Options { + signature=MD5 + fstype = ext4 + } + File = <@tmpdir@/file-list + } +} + + +# List of files to be backed up +FileSet { + Name = "FSTypeFSEmpty" + Include { + Options { + signature=MD5 + fstype = debugfs + } + File = <@tmpdir@/file-list + } +} + FileSet { Name = "FSno5" Include { Options { verify=s5 } @@ -442,6 +498,17 @@ FileSet { } } +FileSet { + Name = "HardlinkSet" + Include { + Options { + signature=MD5 + hardlinks=yes + } + File = <@tmpdir@/file-list + } +} + FileSet { Name = "LZOSet" Include { diff --git a/regress/scripts/prepare-fake-autochanger.in b/regress/scripts/prepare-fake-autochanger.in index 9bf81443ba..75e18095ee 100644 --- a/regress/scripts/prepare-fake-autochanger.in +++ b/regress/scripts/prepare-fake-autochanger.in @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # This script will prepare a big dummy autochanger # @@ -37,6 +35,15 @@ EOF # turn on ach debug touch $adir/log +# the standard script is looking for a real sg device +sed -i 's/test -c /test -f/g' $bin/storage-ctl 2> /dev/null +sed -i 's/test -c /test -f/g' $bin/storage2-ctl 2> /dev/null + +# create generic device for bsg_persist +for i in `seq 0 8`; do + touch $adir/sg$i c +done + nb_vol=`expr $nb_slot - 5` # create $nb_vol volumes for i in `seq 1 $nb_vol`; do diff --git a/regress/scripts/prepare-other-loc b/regress/scripts/prepare-other-loc index e70b6e6b1e..e00105be62 100755 --- a/regress/scripts/prepare-other-loc +++ b/regress/scripts/prepare-other-loc @@ -1,10 +1,9 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - . scripts/functions SCR=$cwd/bin/bacula-ctl @@ -20,4 +19,3 @@ if [ x$plugins != x ]; then mv $cwd/bin/plugins $cwd/bin/plugins.org ln -s $plugins $cwd/bin/plugins fi - diff --git a/regress/scripts/prepare-other-loc2 b/regress/scripts/prepare-other-loc2 new file mode 100755 index 0000000000..d6b7b724bd --- /dev/null +++ b/regress/scripts/prepare-other-loc2 @@ -0,0 +1,27 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# + +. scripts/functions + +sed "s:SCRIPTDIR=/opt/bacula/scripts:SCRIPTDIR=$cwd/bin:" $cwd/bin/bacula > tmp/1 +mv tmp/1 $cwd/bin/bacula +SCR=$cwd/bin/bacula-ctl +sed "s:BACDIRCFG=/opt/bacula/etc:BACDIRCFG=$cwd/bin:" $SCR-dir > tmp/2 +sed "s:PIDDIR=/opt/bacula/working:PIDDIR=$working:" tmp/2 > tmp/1 +mv tmp/1 $SCR-dir +sed "s:BACSDCFG=/opt/bacula/etc:BACSDCFG=$cwd/bin:" $SCR-sd > tmp/2 +sed "s:PIDDIR=/opt/bacula/working:PIDDIR=$working:" tmp/2 > tmp/1 +mv tmp/1 $SCR-sd +sed "s:BACFDCFG=/opt/bacula/etc:BACFDCFG=$cwd/bin:" $SCR-fd > tmp/2 +sed "s:PIDDIR=/opt/bacula/working:PIDDIR=$working:" tmp/2 > tmp/1 +mv tmp/1 $SCR-fd +rm -f tmp/2 +chmod -x $cwd/bin/bacula-fd $cwd/bin/bacula-sd $cwd/bin/bacula-dir $cwd/bin/bconsole +chmod +x $SCR-* $cwd/bin/bacula +if [ x$plugins != x ]; then + mv $cwd/bin/plugins $cwd/bin/plugins.org + ln -s $plugins $cwd/bin/plugins +fi diff --git a/regress/scripts/regress-config.in b/regress/scripts/regress-config.in index d31cd0e60d..6e9f6fb3d9 100755 --- a/regress/scripts/regress-config.in +++ b/regress/scripts/regress-config.in @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # # This is the configuration script for regression testing @@ -45,6 +45,7 @@ fi ${OPENSSL} \ ${TCPWRAPPERS} \ ${WHICHDB} \ + ${CONFIGURE_EXTRA} \ --with-baseport=${BASEPORT} LD_LIBRARY_PATH=${1}/bin:${LD_LIBRARY_PATH} diff --git a/regress/scripts/setup b/regress/scripts/setup index a36f303d27..549de86d3a 100755 --- a/regress/scripts/setup +++ b/regress/scripts/setup @@ -1,10 +1,11 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald -# License: BSD 2-Clause; see file LICENSE-FOSS -# # Script to setup running Bacula regression tests # +# Copyright (C) 2000-2016 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# + check_exit_code() { if [ $? != 0 ] ; then @@ -16,6 +17,8 @@ check_exit_code() . ./config cwd=`pwd` +tmp=${tmp:-"$cwd/tmp"} +mkdir -p $tmp if [ ! -d ${BACULA_SOURCE} ] ; then echo "The BACULA_SOURCE environment variable must be a Bacula release directory, but is not." echo " " @@ -31,11 +34,13 @@ rm -rf build bin echo "Copying source from ${BACULA_SOURCE}" cp -rp ${BACULA_SOURCE} build check_exit_code -which git >/dev/null 2>/dev/null -if [ $? = 0 ] ; then - git show HEAD | grep -m 1 commit | awk '{print $2}' > build/git-version -else - echo "Git not found." >build/git-version +if [ x$PREBUILT != xyes ]; then + which git >/dev/null 2>/dev/null + if [ $? = 0 ] ; then + git show HEAD | grep -m 1 commit | awk '{print $2}' > build/git-version + else + echo "Git not found." >build/git-version + fi fi scripts/create_sed # In default bacula-dir.conf.in, change default file @@ -57,14 +62,17 @@ chmod 755 scripts/regress-config cp scripts/regress-config build cd build rm -f Makefile config.* -# Run Bacula configuration, make, install +# Run Bacula configure ./regress-config ${cwd} -check_exit_code -# Cleanup any build in source -make clean -make ${MAKEOPT} -check_exit_code -make install +if [ x$PREBUILT != xyes ]; then + # Run Bacula make, install + check_exit_code + # Cleanup any build in source + make clean + make ${MAKEOPT} + check_exit_code + make install +fi if [ a${KEEP_SOURCE} = a ]; then rm -rf examples patches check_exit_code @@ -73,9 +81,42 @@ if [ a${KEEP_SOURCE} = a ]; then rm -rf src/win32 examples patches fi # get all tools -- especially testls -cd src/tools -make installall -check_exit_code +if [ x$PREBUILT = xyes ]; then + # populate our bin with start/stop scripts + # and catalog manipulation scripts + cd ${cwd} + mkdir bin + cp ${bin}/../scripts/bacula ${cwd}/bin + cp ${bin}/../scripts/bacula-ctl-* ${cwd}/bin + cp ${bin}/../scripts/create_*_database ${cwd}/bin + cp ${bin}/../scripts/drop_*_database ${cwd}/bin + cp ${bin}/../scripts/drop_*_tables ${cwd}/bin + cp ${bin}/../scripts/make_*_tables ${cwd}/bin + cp ${bin}/../scripts/grant_*_privileges ${cwd}/bin + cp ${bin}/../scripts/storage*-ctl ${cwd}/bin + cp ${bin}/../scripts/storage*-ctl.conf ${cwd}/bin + cp ${bin}/../scripts/mtx-changer ${cwd}/bin + cp ${bin}/../scripts/mtx-changer.conf ${cwd}/bin + cp ${bin}/bconsole ${cwd}/bin + cd ${cwd}/bin + for i in create_*_database drop_*_database drop_*_tables make_*_tables grant_*_privileges; do + echo "Doing $i" + sed "s/db_name:-bacula/db_name:-regress/" $i >1 + sed "s/db_name=bacula/db_name=regress/" 1 >2 + sed "s/db_user:-bacula/db_user:-regress/" 2 >3 + sed "s%/opt/bacula/scripts%${cwd}/bin%" 3 >4 + mv 4 $i + chmod +x $i + rm -f 1 2 3 + done + cd ${cwd} + scripts/prepare-other-loc2 +else + cd src/tools + make installall + check_exit_code +fi + cd ${cwd} bin/bacula stop -KILL @@ -92,7 +133,15 @@ cd ${cwd} bin/bacula start bin/bacula stop # -# Save Bacula default conf files for later use +# Save Bacula default conf files into scripts for later use # -cp -f bin/*.conf scripts -check_exit_code +if [ x$PREBUILT = xyes ]; then + cp build/src/console/*.conf scripts + cp build/src/dird/*.conf scripts + cp build/src/filed/*.conf scripts + cp build/src/stored/*.conf scripts +else + # these were installed so take them from bin + cp -f bin/*.conf scripts + check_exit_code +fi diff --git a/regress/scripts/test-bacula-sd.conf.in b/regress/scripts/test-bacula-sd.conf.in index 0a54d72bc1..5f92f7a6a2 100644 --- a/regress/scripts/test-bacula-sd.conf.in +++ b/regress/scripts/test-bacula-sd.conf.in @@ -57,6 +57,18 @@ Device { AlwaysOpen = no; } +Device { + Name = Dummy + Media Type = Dummy + Device Type = Fifo + Archive Device = /dev/null + LabelMedia = yes + Random Access = no + AutomaticMount = no + RemovableMedia = no + MaximumOpenWait = 60 + AlwaysOpen = no +} #Device { # Name = "HP DLT 80" @@ -107,3 +119,14 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/tls-auth-bacula-sd.conf.in b/regress/scripts/tls-auth-bacula-sd.conf.in index 880effdb27..e46a0c0059 100644 --- a/regress/scripts/tls-auth-bacula-sd.conf.in +++ b/regress/scripts/tls-auth-bacula-sd.conf.in @@ -115,3 +115,14 @@ Messages { Name = Standard director = @hostname@-dir = all, !terminate } +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} + diff --git a/regress/scripts/tls-bacula-sd.conf.in b/regress/scripts/tls-bacula-sd.conf.in index 6fef8b7bdf..34715a6fb7 100644 --- a/regress/scripts/tls-bacula-sd.conf.in +++ b/regress/scripts/tls-bacula-sd.conf.in @@ -113,3 +113,14 @@ Messages { Name = Standard director = localhost-dir = all, !terminate } + +Cloud { + Name = DummyCloud + Driver = "File" + HostName = "@tmpdir@/cloud" + BucketName = "DummyBucket" + AccessKey = "DummyAccessKey" + SecretKey = "DummySecretKey" + Protocol = HTTPS + UriStyle = VirtualHost +} diff --git a/regress/scripts/tls-crypto-bacula-fd.conf.in b/regress/scripts/tls-crypto-bacula-fd.conf.in new file mode 100644 index 0000000000..2369f518ee --- /dev/null +++ b/regress/scripts/tls-crypto-bacula-fd.conf.in @@ -0,0 +1,47 @@ +# +# Default Bacula File Daemon Configuration file +# +# For Bacula release 2.0 +# +# There is not much to change here except perhaps the +# File daemon Name to +# + +# +# List Directors who are permitted to contact this File daemon +# +Director { + Name = localhost-dir + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" + TLS Require = yes + TLS Certificate = "@scriptdir@/tls-cert.pem" + TLS Key = "@scriptdir@/tls-cert.pem" + TLS CA Certificate File = "@scriptdir@/tls-CA.pem" +} + +# +# "Global" File daemon configuration specifications +# +FileDaemon { # this is me + Name = localhost-fd + FDPort = @fdport@ # where we listen for the director + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + SubSys Directory = "@subsysdir@" + Plugin Directory = "@sbindir@/plugins" + Maximum Concurrent Jobs = 100 + TLS Require = yes + TLS Certificate = "@scriptdir@/tls-cert.pem" + TLS Key = "@scriptdir@/tls-cert.pem" + TLS CA Certificate File = "@scriptdir@/tls-CA.pem" + PKI Signatures = Yes + PKI Encryption = Yes + PKI Keypair = "@scriptdir@/cryptokeypair.pem" + PKI Master Key = "@scriptdir@/master2048.cert" +} + +# Send all messages except skipped files back to Director +Messages { + Name = Standard + director = localhost-dir = all, !terminate, !restored +} diff --git a/regress/scripts/virtualfull-extreme-bacula-dir.conf.in b/regress/scripts/virtualfull-extreme-bacula-dir.conf.in index 9f1b382d28..7acf794170 100644 --- a/regress/scripts/virtualfull-extreme-bacula-dir.conf.in +++ b/regress/scripts/virtualfull-extreme-bacula-dir.conf.in @@ -655,9 +655,9 @@ Pool { Name = "Hot1" Pool Type = Backup Storage = "VirtB" - Job Retention = 2 hours - File Retention = 100 min - Volume Retention = 3 hours + Job Retention = 1 year + File Retention = 1 year + Volume Retention = 1 year Maximum Volume Jobs = 7 Label Format = "Hot1-" Next Pool = "Hot2" @@ -667,9 +667,9 @@ Pool { Name = "Hot2" Pool Type = Backup Storage = "VirtB" - Job Retention = 2 hours - File Retention = 100 min - Volume Retention = 3 hours + Job Retention = 1 year + File Retention = 1 year + Volume Retention = 1 year Maximum Volume Jobs = 7 Label Format = "Hot2-" Next Pool = "Hot3" @@ -679,9 +679,9 @@ Pool { Name = "Hot3" Pool Type = Backup Storage = "VirtA" - Job Retention = 2 hours - File Retention = 105 min - Volume Retention = 2.5 hours + Job Retention = 1 year + File Retention = 1 year + Volume Retention = 1 year Maximum Volume Jobs = 1 Label Format = "Hot3-" } @@ -700,6 +700,16 @@ Job { Max Run Time = 30min } +Job { + Name = VerifyData + Type = Verify + Level = Data + Client = HotV01-fd + FileSet = "Full Set" + Storage = File + Messages = Standard + Pool = Hot1 +} # List of files to be backed up FileSet { diff --git a/regress/tests/2drive-offline-test b/regress/tests/2drive-offline-test index 498154f270..9351ccf924 100755 --- a/regress/tests/2drive-offline-test +++ b/regress/tests/2drive-offline-test @@ -1,9 +1,4 @@ #!/bin/sh -# -# Copyright (C) 2000-2015 Kern Sibbald -# License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Set prefer mounted volumes to no to try to force use of # the drive swap code. @@ -18,7 +13,7 @@ JobName="2driveoffline" . scripts/functions scripts/cleanup -scripts/copy-2disk-drive-confs +scripts/copy-2client-confs scripts/prepare-disk-changer CLIENT=2drive2disk @@ -35,6 +30,9 @@ cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 # otherwise, it writes the two jobs to different drives sed -f ${outf} ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf +# One FD is not reachable, try to speed up the test +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "FdConnectTimeout", "10", "Director")' + # Write out bconsole commands cat <${cwd}/tmp/bconcmds @$out /dev/null @@ -43,16 +41,18 @@ messages label storage=tape volume=TestVolume001 slot=2 Pool=Default drive=0 label storage=tape volume=TestVolume002 slot=1 Pool=Default drive=1 status storage=tape -setdebug level=120 storage=tape -run job=Offline level=Full yes +@#setdebug level=120 storage=tape +run job=Offline comment="This job will fail, the client is not here" level=Full yes @sleep 2 -run job=$JobName level=Full yes +run job=$JobName spooldata=no level=Full yes status storage=tape wait list volumes list jobs -status storage=tape messages +@$out ${cwd}/tmp/log3.out +@# TODO: We see that one volume is still reserved while the job is not here +status storage=tape quit END_OF_DATA @@ -68,7 +68,7 @@ messages @#unmount storage=tape drive=1 @#mount storage=tape slot=1 drive=0 @#mount storage=tape slot=2 drive=1 -restore where=${cwd}/tmp/bacula-restores select all storage=tape done +restore where=${cwd}/tmp/bacula-restores client=$HOST-fd select all storage=tape done yes wait messages @@ -76,10 +76,30 @@ quit END_OF_DATA run_bconsole -check_for_zombie_jobs storage=tape +check_for_zombie_jobs storage=tape client=$HOST-fd stop_bacula -check_two_logs +grep 'Reserved volume: TestVolume002 on File device "Drive-0"' $tmp/log1.out +if [ $? -eq 0 ]; then + print_debug "ERROR: TestVolume001 should be used on Drive-0, not TestVolume002" + estat=1 +fi + +grep 'Reserved volume: TestVolume001 on File device "Drive-1"' $tmp/log1.out +if [ $? -eq 0 ]; then + print_debug "ERROR: TestVolume002 should be used on Drive-1, not TestVolume001" + estat=1 +fi + +grep 'No Jobs running.' $tmp/log3.out > /dev/null +if [ $? -eq 0 ]; then + grep 'Reserved volume: TestVolume001 on File device "Drive-0"' $tmp/log3.out + if [ $? -eq 0 ]; then + print_debug "WARNING: TestVolume001 is still reserved on Drive-0 in $tmp/log3.out" + #estat=1 + fi +fi + check_restore_diff end_test diff --git a/regress/tests/2media-virtual-test b/regress/tests/2media-virtual-test new file mode 100755 index 0000000000..32ea1a7961 --- /dev/null +++ b/regress/tests/2media-virtual-test @@ -0,0 +1,71 @@ +#!/bin/sh +# +# Test running a Virtual disk changer with two Media Types +# and two different Archive directories +# This tests bug #8033 where a Volume is not truncated, +# but a second Volume of the same name is made in the wrong +# directory. +# +TestName="2media-virtual-test" +JobName=Virtual +. scripts/functions + +${rscripts}/cleanup +cp -f ${rscripts}/bacula-dir-2media-virtual.conf ${conf}/bacula-dir.conf +cp -f ${rscripts}/bacula-sd-2media-virtual.conf ${conf}/bacula-sd.conf +cp -f ${rscripts}/test-bacula-fd.conf ${conf}/bacula-fd.conf +rm -rf ${tmp}/disk ${tmp}/disk1 +mkdir -p ${tmp}/disk ${tmp}/disk1 + +disable_plugins +echo "$cwd/build/src/dird" >${cwd}/tmp/file-list + +start_test + +cat <${tmp}/bconcmds +@output /dev/null +messages +@$out ${tmp}/log1.out +@#setdebug level=15 storage=Virtual +@#setdebug level=200 client +@#setdebug level=100 director +label storage=Virtual pool=Default volume=TestVolume001 drive=0 slot=0 +label Storage=vDrive-3 pool=Default volume=TestVolume002 drive=2 slot=0 +run job=$JobName level=Full storage=Virtual yes +wait +update volume=TestVolume001 volstatus=Used actiononpurge=truncate +run job=$JobName level=Full storage=vDrive-3 yes +wait +messages +list volumes +llist volume=TestVolume001 +purge volume=TestVolume001 +update volume=TestVolume002 volstatus=Used actiononpurge=truncate +purge volume=TestVolume002 +purge volume action=truncate pool=Default storage=Virtual drive=0 +purge volume action=truncate pool=Default storage=Virtual drive=2 +list volumes +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=Virtual +stop_bacula + +# No TestVolume002 should be in ${tmp}/disk +ls -l ${tmp}/disk | grep -q TestVolume002 +if [ $? = 0 ]; then + print_debug "ERROR: Volume TestVolume2 incorrectly created in ${tmp}/disk" + dstat=1 +fi + +if [ x$REGRESS_DEBUG != x ] ; then + ls -l ${tmp}/disk + ls -l ${tmp}/disk1 +fi + +exit + +#check_two_logs +#check_restore_diff +end_test diff --git a/regress/tests/accurate-test b/regress/tests/accurate-test index 096254458f..c922f70662 100755 --- a/regress/tests/accurate-test +++ b/regress/tests/accurate-test @@ -415,7 +415,7 @@ jobid=`awk '/ Incr.+backup/ { jobid=$1 } END { print jobid }' ${cwd}/tmp/log3.ou cat <${cwd}/tmp/bconcmds @$out ${cwd}/tmp/log3.out -list files jobid=$jobid +list files type=all jobid=$jobid quit END_OF_DATA diff --git a/regress/tests/action-on-purge-test b/regress/tests/action-on-purge-test index c288a32422..0d013bdcf8 100755 --- a/regress/tests/action-on-purge-test +++ b/regress/tests/action-on-purge-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # TestName="action-on-purge-test" @@ -100,40 +98,54 @@ run_bacula check_for_zombie_jobs storage=File print_debug "Check all media" -perl -Mscripts::functions -e 'foreach $i (1..5){check_min_volume_size(4096,"TestVolume00$i")}' - -if [ $? != 0 ]; then - print_debug `ls -l $tmp` - print_debug "Initialization problem" - estat=2 -fi +for i in 1 2 3 4 5 ; do + size=`du -b $tmp/TestVolume00$i|cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 2: Volume TestVolum00$i is too small $size" + ls -l $tmp/TestVolume00$i + estat=2 + fi +done echo "purge volume=TestVolume001 action storage=File" > $tmp/bconcmds run_bconsole -perl -Mscripts::functions -e ' -check_min_volume_size(4096, "TestVolume002","TestVolume003","TestVolume004","TestVolume005"); -check_max_volume_size(4096, "TestVolume001")' - -if [ $? != 0 ]; then - print_debug `ls -l $tmp` - ls -l $tmp - estat=2 -fi +for i in 2 3 4 5 ; do + size=`du -b $tmp/TestVolume00$i|cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 3: Volume TestVolum00$i is too small $size" + ls -l $tmp/TestVolume00$i + estat=3 + fi +done +for i in 1 ; do + size=`du -b $tmp/TestVolume00$i|cut -f1` + if test $size -gt 5000 ; then + print_debug "ERROR 4: Volume TestVolum00$i is not truncated (too big) $size" + ls -l $tmp/TestVolume00$i + estat=4 + fi +done echo "purge volume action storage=File" > $tmp/bconcmds run_bconsole -perl -Mscripts::functions -e ' -check_min_volume_size(4096, "TestVolume002", "TestVolume005"); -check_max_volume_size(4096, "TestVolume001", "TestVolume003", "TestVolume004")' - -if [ $? != 0 ]; then - print_debug `ls -l $tmp` - ls -l $tmp - estat=2 -fi - +for i in 2 5 ; do + size=`du -b $tmp/TestVolume00$i|cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 5: Volume TestVolum00$i is too small $size" + ls -l $tmp/TestVolume00$i + estat=5 + fi +done +for i in 1 3 4 ; do + size=`du -b $tmp/TestVolume00$i|cut -f1` + if test $size -gt 5000 ; then + print_debug "ERROR 6: Volume TestVolum00$i is not truncated (too big) $size" + ls -l $tmp/TestVolume00$i + estat=6 + fi +done cat < $tmp/bconcmds @######################################################### diff --git a/regress/tests/aligned-bug-8013-test b/regress/tests/aligned-bug-8013-test new file mode 100755 index 0000000000..44003e27ca --- /dev/null +++ b/regress/tests/aligned-bug-8013-test @@ -0,0 +1,131 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test of aligned volumes. It does not require running on +# a ZFS filesystem, but attempts to reproduce a condition +# where the volume and catalog sizes do not agree, and +# when Bacula fixes it another number of Files error occurs +# causing the second job to fail. +# +TestName="aligned-bug-8013-test" +JobName=NightlySave +. scripts/functions + +if test x$FORCE_CLOUD = xyes ; then + echo "\n=== Test $TestName skipped not compatible with Cloud ===" + exit 0 +fi + +scripts/cleanup +scripts/copy-test-confs +cp scripts/aligned-bacula-sd.conf bin/bacula-sd.conf + +# install the aligned volume plugin +#make -C build/src/plugins/sd install-aligned-plugin >/dev/null + +echo "${cwd}/build" >${cwd}/tmp/file-list + +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=Vol1 +label storage=File volume=Vol2 +label storage=File volume=Vol3 +label storage=File volume=Vol4 +label storage=File volume=Vol5 +update volume=Vol1 maxvolbytes=60M +update volume=Vol2 maxvolbytes=60M +update volume=Vol3 maxvolbytes=60M +update volume=Vol4 maxvolbytes=60M +@# Leave Vol5 with no size limit +END_OF_DATA + +# do label +run_bacula + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +setdebug level=0 hangup=100 client +run job=$JobName level=Full yes +setdebug level=0 hangup=0 client +wait +list volumes +update volume=Vol1 volstatus="Append" +sql +update Media SET VolFiles=2,VolBytes=100,VolABytes=100 WHERE VolumeName='Vol1'; + +setdebug level=15 storage=File +list volumes +messages +END_OF_DATA + +for i in 1 2 3 4 5 ; do + if test "$debug" -eq 1 ; then + echo "Running job $i" + fi + run_bconsole + if test "$debug" -eq 1 ; then + ls -l tmp/Vol* + du -h tmp/Vol1.add + fi +done + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +setdebug level=0 hangup=0 client +run job=$JobName level=Full yes +wait +list volumes +messages +END_OF_DATA + +for i in 6 7 8 9 10; do + if test "$debug" -eq 1 ; then + echo "Running job $i" + fi + run_bconsole + if test "$debug" -eq 1 ; then + ls -l tmp/Vol* + du -h tmp/Vol1.add + fi +done + +cat <${cwd}/tmp/bconcmds +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +@# setdebug level=0 fd +restore where=${cwd}/tmp/bacula-restores storage=File +5 +mark * +done +yes +wait +messages +quit +END_OF_DATA + +run_bconsole + +if test "$debug" -eq 1 ; then + ls -l tmp/Vol* + du -h tmp/Vol1.add +fi + +sleep 2 +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/aligned-test b/regress/tests/aligned-test new file mode 100755 index 0000000000..ea54a55b33 --- /dev/null +++ b/regress/tests/aligned-test @@ -0,0 +1,147 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test aligned data volumes with ZFS deduplication +# This test expects that you have created a ZFS pool named +# tank, and that you have also created tank/volumes +# +TestName="aligned-test" +JobName=NightlySave +. scripts/functions + +if test x$FORCE_CLOUD = xyes ; then + echo "\n=== Test $TestName skipped not compatible with Cloud ===" + exit 0 +fi + +scripts/cleanup +scripts/copy-test-confs +cp scripts/aligned-bacula-sd.conf bin/bacula-sd.conf + +# install the aligned volume plugin +#make -C build/src/plugins/sd install-aligned-plugin >/dev/null + +echo "${cwd}/build" >${cwd}/tmp/file-list + +which zfs >/dev/null 2>&1 +if [ $? -eq 0 ] ; then + zfs=1 +else + zfs=0 +fi + +# If we are running with zfs, setup a pool, +# otherwise just use defaults +if [ $zfs -eq 1 ]; then + cp bin/bacula-sd.conf tmp/1 + sed "s%Archive Device = .*$%Archive Device = /tank/volumes%g" tmp/1 >bin/bacula-sd.conf + + # Delete any previous zfs pool + sudo zfs destroy -fR tank/volumes + sudo zfs destroy -fR tank + sudo zfs destroy -r tank + sudo zpool destroy -f tank + sudo rm -rf /tank/volumes + sudo rm -rf /tank + # + # Create zfs pool + sudo zpool create tank sdb sdc + sudo zfs create tank/volumes + #sudo zfs recordsize=64k tank # default is 128K + sudo zfs set atime=off tank + sudo zfs set compress=on tank + sudo zfs set dedup=on tank + sudo zdb -dd tank + sudo chown -R kern:kern /tank +fi + +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=Vol1 +label storage=File volume=Vol2 +label storage=File volume=Vol3 +END_OF_DATA + +# do label +run_bacula + +if [ $zfs -eq 1 ] ; then + sudo zdb -DD tank + sudo zdb -b tank + sudo zpool list tank +fi + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +setdebug level=200 storage +run job=$JobName level=Full yes +wait +list volumes +messages +END_OF_DATA + +for i in 1 2 3 4 5 6 7 8 9 10; do + #echo "Running job $i" + run_bconsole + if [ $zfs -eq 1 ] ; then + sudo zdb -DD tank + sudo zdb -b tank + sudo zpool list tank + ls -l /tank/volumes/Vol* >${cwd}/tmp/log5.out + du -h /tank/volumes/Vol1.add >>${cwd}/tmp/log5.out + else + ls -l tmp/Vol* >${cwd}/tmp/log5.out + du -h tmp/Vol1.add >>${cwd}/tmp/log5.out + fi +done + +cat <${cwd}/tmp/bconcmds-form +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +@# setdebug level=0 fd +restore where=${cwd}/tmp/bacula-restores storage=File jobid=@jobid@ +mark * +done +yes +wait +messages +quit +END_OF_DATA + +# Restore and check each job +for i in 1 2 3 4 5 6 7 8 9 10; do + rm -rf ${cwd}/tmp/bacula-restores + echo "s%@jobid@%$i%" >${cwd}/tmp/in + sed -f ${cwd}/tmp/in ${cwd}/tmp/bconcmds-form >${cwd}/tmp/bconcmds + run_bconsole + check_restore_diff +done + +if [ $zfs -eq 1 ] ; then + sudo zdb -DD tank + sudo zdb -b tank + sudo zpool list tank + ls -l /tank/volumes/Vol* >>${cwd}/tmp/log5.out + du -h /tank/volumes/Vol1.add >>${cwd}/tmp/log5.out +else + ls -l tmp/Vol* >>${cwd}/tmp/log5.out + du -h tmp/Vol1.add >>${cwd}/tmp/log5.out +fi + +sleep 2 +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/ansi-label-tape b/regress/tests/ansi-label-tape index 08af0683e7..34beef734a 100755 --- a/regress/tests/ansi-label-tape +++ b/regress/tests/ansi-label-tape @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Test of ANSI labeled tapes # @@ -33,7 +31,7 @@ cat <${cwd}/tmp/bconcmds @$out /dev/null messages @$out ${cwd}/tmp/log1.out -setdebug level=200 storage=tape +@#setdebug level=200 storage=tape label storage=tape volume=Vol001 slot=0 pool=Default purge volume=Vol001 relabel pool=Default storage=tape oldVolume=Vol001 volume=Vol002 slot=0 diff --git a/regress/tests/auto-label-jobmedia-test b/regress/tests/auto-label-jobmedia-test new file mode 100755 index 0000000000..6ff5c699ef --- /dev/null +++ b/regress/tests/auto-label-jobmedia-test @@ -0,0 +1,109 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Try to stress test autolabel by creating lots of new +# volumes. This is to try to show up a bug 8106 where +# at some point when a volume is changed, the JobMedia +# EndFile < StartFile and EndBlock < StartBlock because +# it is picking up the information from the next Volume. +# Note, this does not reproduce the error ... +# + +TestName="auto-label-jobmedia-test" +JobName=AutoLabel +. scripts/functions + +copy_test_confs + +echo "${cwd}/build" >${cwd}/tmp/file-list + +# Default values +# Maximum Volume Jobs = 1 +# Maximum Volume Bytes = 1500000 + +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%# Simple Label Format% Label Format%" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%# Maximum Volume% Maximum Volume%g" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%Maximum Volume Jobs = 1%Maximum Volume Jobs = 10%g" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%Maximum Volume Bytes = 1500000%Maximum Volume Bytes =45000000%g" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf + +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%Maximum Concurrent Jobs = 10%Maximum Concurrent Jobs = 100%g" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%Maximum Concurrent Jobs = 4%Maximum Concurrent Jobs = 100%g" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf + + +change_jobname CompressedTest $JobName +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +messages +@#setdebug level=50 storage=File +@#setdebug level=50 dir +run job=$JobName level=Full storage=File yes +run job=$JobName storage=File yes +run job=$JobName level=Full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +run job=$JobName level=full storage=File yes +status dir +wait +sql +select * from JobMedia; + +list volumes +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +setdebug level=0 storage=File +setdebug level=0 dir +restore where=${cwd}/tmp/bacula-restores select storage=File +unmark * +mark * +done +yes +wait +messages +@# Now restore JobId=1 +restore where=${cwd}/tmp/bacula-restores storage=File +3 +1 +mark * +done +yes +wait +messages +@# Now restore JobId=3 +restore where=${cwd}/tmp/bacula-restores storage=File +3 +3 +mark * +done +yes +wait +messages +quit +END_OF_SCRIPT + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/auto-label-many-test b/regress/tests/auto-label-many-test new file mode 100755 index 0000000000..3bd946364a --- /dev/null +++ b/regress/tests/auto-label-many-test @@ -0,0 +1,90 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# +# Try to stress test autolabel by creating lots of new +# volumes. This is to try to show up a bug 8103 where +# at some point a volume is created in the catalog but +# not labeled, and then produces errors. +# + +TestName="auto-label-many-test" +JobName=AutoLabel +. scripts/functions + +copy_test_confs + +echo "${cwd}/build" >${cwd}/tmp/file-list + +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%# Simple Label Format% Label Format%" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +sed "s%# Maximum Volume% Maximum Volume%g" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf + +change_jobname CompressedTest $JobName +start_test + +#$bperl -e 'add_attribute("$conf/bacula-dir.conf", "SpoolData", "no", "Job")' +#$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Upload", "no", "Cloud")' + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +messages +setdebug level=200 trace=1 options=hT storage=File +@#setdebug level=50 dir +run job=$JobName level=Full storage=File yes +run job=$JobName storage=File yes +run job=$JobName level=Full storage=File yes +run job=$JobName storage=File yes +@#run job=$JobName storage=File yes +@#run job=$JobName storage=File yes +@#run job=$JobName storage=File yes +status dir +wait +list volumes +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +setdebug level=0 storage=File +setdebug level=0 dir +restore where=${cwd}/tmp/bacula-restores select storage=File +unmark * +mark * +done +yes +wait +messages +@# Now restore JobId=1 +restore where=${cwd}/tmp/bacula-restores storage=File +3 +1 +mark * +done +yes +wait +messages +@# Now restore JobId=3 +restore where=${cwd}/tmp/bacula-restores storage=File +3 +3 +mark * +done +yes +wait +messages +quit +END_OF_SCRIPT + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/auto-label-test b/regress/tests/auto-label-test index 30c3b7992c..9c45d96efb 100755 --- a/regress/tests/auto-label-test +++ b/regress/tests/auto-label-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Test if Bacula can automatically create a Volume label. # diff --git a/regress/tests/backup-to-null b/regress/tests/backup-to-null index afc948ac32..7e00e14d6f 100755 --- a/regress/tests/backup-to-null +++ b/regress/tests/backup-to-null @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a simple backup of the Bacula build directory # to /dev/null Of course, it's not possible to restore ;-) @@ -14,6 +12,11 @@ JobName=backuptonull . scripts/functions +# Not compatible with a backup to /dev/null +unset FORCE_DEDUP +unset FORCE_ALIGNED +unset FORCE_CLOUD + scripts/cleanup scripts/copy-fifo-confs diff --git a/regress/tests/base-job-test b/regress/tests/base-job-test index abcebac1ee..1048a7a43e 100755 --- a/regress/tests/base-job-test +++ b/regress/tests/base-job-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a basejob backup of the Bacula build directory # then restore it. @@ -112,7 +110,7 @@ if [ $? -ne 0 ]; then bstat=2 fi -grep -e 'Using Base JobId(s): 1$' ${cwd}/tmp/log4.out > /dev/null +grep -e 'Using BaseJobId(s): 1$' ${cwd}/tmp/log4.out > /dev/null if [ $? -ne 0 ]; then print_debug "ERROR: The first full job should use only jobid=1 as basejob" bstat=2 diff --git a/regress/tests/bls-test b/regress/tests/bls-test new file mode 100755 index 0000000000..8df11a1516 --- /dev/null +++ b/regress/tests/bls-test @@ -0,0 +1,54 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a simple backup of the Bacula build directory then bls +# the resulting Volume +# +TestName="bls-test" +JobName=Simple +. scripts/functions + +${rscripts}/cleanup +${rscripts}/copy-test-confs +echo "${tmpsrc}" >${tmp}/file-list +mkdir -p ${tmpsrc} +cp -p ${src}/src/dird/*.c ${tmpsrc} +cd ${tmp} +echo "${tmpsrc}/ficheriro1.txt" >restore-list +echo "${tmpsrc}/ficheriro2.txt" >>restore-list +cd ${cwd} + +start_test + +cat <${tmp}/bconcmds +@$out /dev/null +messages +@$out ${tmp}/log1.out +@#setdebug level=200 storage=File +@#setdebug level=200 client +@#setdebug level=100 director +label storage=File volume=TestVolume001 +label storage=File volume=TestVolume002 +run job=$JobName yes +status client +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +$bin/bls -d 50 -v -c $conf/bacula-sd.conf -V 'TestVolume001' FileStorage > $tmp/bls.out +if [ $? != 0 ] ; then + zstat=2 +fi +if test "$debug" -eq 1 ; then + cat $tmp/bls.out +fi + +#check_restore_tmp_build_diff +#end_test diff --git a/regress/tests/bpipe-test b/regress/tests/bpipe-test new file mode 100755 index 0000000000..ac21afd814 --- /dev/null +++ b/regress/tests/bpipe-test @@ -0,0 +1,160 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# +# Attempt to backup and restore a file with the bpipe plugin +# +TestName="bpipe-test" +JobName=pluginTest +. scripts/functions + +scripts/cleanup +scripts/copy-plugin-confs + +file=encrypt-bug.jpg +rm -rf ${cwd}/tmp/* +echo "${cwd}/README" >${cwd}/tmp/file-list + +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File1 volume=TestVolume001 +estimate job=$JobName level=Full +@#setdebug level=50 traclient=$CLIENT +run job=$JobName storage=File1 yes +wait +@#setdebug level=50 client=$CLIENT +run job=$JobName storage=File1 yes +wait +status client=$CLIENT +messages +quit +END_OF_DATA + + +run_bacula + +cat <${cwd}/tmp/bconcmds +@$out $tmp/files +list files jobid=1 +list files jobid=2 +@$out $tmp/list +llist pluginrestoreconf jobid=1 +llist pluginrestoreconf jobid=2 +@$out $tmp/conf +llist pluginrestoreconf jobid=1 id=1 +llist pluginrestoreconf jobid=1 id=2 +END_OF_DATA + +run_bconsole + +nb=`grep Makefile $tmp/files | wc -l` +if [ $nb -ne 2 ]; then + print_debug "ERROR: Should have two times Makefile in job files $tmp/files" + bstat=1 +fi + +nb=`grep $file $tmp/files | wc -l` +if [ $nb -ne 2 ]; then + print_debug "ERROR: Should have two times $file in job files $tmp/files" + bstat=1 +fi + +nb=`grep README $tmp/files | wc -l` +if [ $nb -ne 1 ]; then + print_debug "ERROR: Should have one time README in job files $tmp/files" + bstat=1 +fi + +nb=`grep $file $tmp/list | wc -l` +if [ $nb -ne 1 ]; then + print_debug "ERROR: Should find the RestoreObject for $file in $tmp/list" + bstat=1 +fi + +nb=`grep Makefile $tmp/list | wc -l` +if [ $nb -ne 1 ]; then + print_debug "ERROR: Should find the RestoreObject for Makefile in $tmp/list" + bstat=1 +fi + +nb=`grep restore_command $tmp/conf | wc -l` +if [ $nb -ne 2 ]; then + print_debug "ERROR: Should find the RestoreObject for Makefile and $file in $tmp/conf" + bstat=1 +fi + +cat <$tmp/obj +restore_command="cat >$tmp/Makefile.bak" +EOF + +cat <${cwd}/tmp/bconcmds +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log3.out +setdebug level=50 client=$CLIENT trace=1 +@putfile obj1 $tmp/obj +restore pluginrestoreconf="2:obj1" where=${cwd}/tmp select all storage=File1 done +yes +wait +setdebug level=0 client=$CLIENT trace=0 +messages +quit +END_OF_DATA + +run_bconsole + +# +# Remove plugin so we can try the restore without the plugin +# +mv -f ${cwd}/bin/plugins/bpipe-fd.so ${cwd}/bin/plugins/bpipe-fd.sox + +cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log2.out +@# +@# now do a restore without the plugin +@# +@$out ${cwd}/tmp/log2.out +@#setdebug level=50 client=$CLIENT +restore where=${cwd}/tmp select all storage=File1 done +yes +wait +messages +quit +END_OF_DATA + +stop_bacula +run_bacula -d50 +run_bconsole + +check_for_zombie_jobs storage=File1 +stop_bacula +# +# Restore plugin +# +mv -f ${cwd}/bin/plugins/bpipe-fd.sox ${cwd}/bin/plugins/bpipe-fd.so + +check_two_logs +# +# ****FIXME**** test that all three files are restored correctly +# +diff ${cwd}/Makefile ${cwd}/tmp/Makefile.bak +dstat=$? + +diff ${cwd}/${file} ${cwd}/tmp/${file} +dstat=$(($dstat + $?)) + +diff ${cwd}/tmp/@bpipe@/${file} ${cwd}/${file} +dstat=$(($dstat + $?)) + +diff ${cwd}/tmp/@bpipe@/Makefile ${cwd}/Makefile +dstat=$(($dstat + $?)) + +end_test diff --git a/regress/tests/bscan-tape b/regress/tests/bscan-tape index 797444a5e3..cf5adc3dc6 100755 --- a/regress/tests/bscan-tape +++ b/regress/tests/bscan-tape @@ -45,7 +45,6 @@ END_OF_DATA run_bacula check_for_zombie_jobs storage=tape -echo "Backup 1 done" # make some files for the incremental to pick up touch ${cwd}/build/src/dird/*.c ${cwd}/build/src/dird/*.o touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o @@ -66,7 +65,6 @@ END_OF_DATA run_bconsole scripts/check_for_zombie_jobs storage=tape -echo "Backup 2 done" touch ${cwd}/build/src/dird/*.c touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o # @@ -76,7 +74,6 @@ touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o run_bconsole scripts/check_for_zombie_jobs storage=tape -echo "Backup 3 done" # make some files for the incremental to pick up touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o #echo "abc" > ${cwd}/build/src/lib/dummy @@ -89,7 +86,6 @@ scripts/check_for_zombie_jobs storage=tape stop_bacula -echo "Backup 4 done" # # now drop and recreate the database # diff --git a/regress/tests/bsr-test b/regress/tests/bsr-test new file mode 100755 index 0000000000..e88f37fed6 --- /dev/null +++ b/regress/tests/bsr-test @@ -0,0 +1,62 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# +# Run a simple backup of the Bacula build directory +# then restore it. Used to generate BSR files. +# +TestName="bsr-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +echo "s% Write Bootstrap%# Write Bootstrap%g" >> ${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumFileSize', '1MB', 'Device')" + +change_jobname BackupClient1 $JobName +start_test + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +run job=$JobName yes +wait +messages +@# +@# now do a restore +@# +@$out $tmp/log2.out +restore where=$tmp/bacula-restores select all done +@exec "sh -c 'cp $working/*.bsr $tmp/1.bsr'" +yes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File1 +stop_bacula + +check_two_logs +check_restore_diff + +# here we can add checks on the BSR file +awk '/Volume=/ { nbvol++ } /FileIndex=/ { nbfdx++ } END { print "Found Volumes=" nbvol " FileIndexes=" nbfdx }' $tmp/1.bsr + +end_test diff --git a/regress/tests/btape-fill-full-changer b/regress/tests/btape-fill-full-changer index b678e8114b..1a6cb1c5f5 100755 --- a/regress/tests/btape-fill-full-changer +++ b/regress/tests/btape-fill-full-changer @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Test the fill command in btape # @@ -32,14 +30,14 @@ sed -e 's%64512%262144%' ${cwd}/tmp/1 >${cwd}/bin/bacula-sd.conf # sed -e 's%# MaximumVolumeSize = 400M% MaximumVolumeSize = 400M%' ${cwd}/tmp/1 >${cwd}/bin/bacula-sd.conf if test "$debug" -eq 1 ; then - $bin/btape -c bin/bacula-sd.conf Drive-0 <${cwd}/tmp/log1.out 2>&1 + $bin/btape -w $tmp -c bin/bacula-sd.conf Drive-0 <${cwd}/tmp/log1.out 2>&1 fill m diff --git a/regress/tests/btape-fill-full-tape b/regress/tests/btape-fill-full-tape index 46cee5c736..af58430f4e 100755 --- a/regress/tests/btape-fill-full-tape +++ b/regress/tests/btape-fill-full-tape @@ -1,9 +1,8 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # - # # Test the fill command in btape # @@ -27,14 +26,14 @@ sed -e 's%64512%262144%' ${cwd}/tmp/1 >${cwd}/bin/bacula-sd.conf if test "$debug" -eq 1 ; then - $bin/btape -c bin/bacula-sd.conf Drive-0 <${cwd}/tmp/log1.out 2>&1 + $bin/btape -w $tmp -c bin/bacula-sd.conf Drive-0 <${cwd}/tmp/log1.out 2>&1 fill s diff --git a/regress/tests/btape-test b/regress/tests/btape-test new file mode 100755 index 0000000000..00c86f9fcc --- /dev/null +++ b/regress/tests/btape-test @@ -0,0 +1,108 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# +# Test the test command in btape +# +TestName="btape-test" +JobName=btapetest +. scripts/functions + +require_vtape + +scripts/cleanup +scripts/copy-tape-confs +cp $rscripts/bacula-dir-vtape.conf $conf/bacula-dir.conf +cp $rscripts/bacula-sd-vtape.conf $conf/bacula-sd.conf +scripts/prepare-fake-autochanger + +# we need at least 2 or 3G for this test +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumVolumeSize', '3G', 'Device')" + +change_jobname $JobName +start_test + +$bin/disk-changer $working/ach load 1 $working/ach/drive0 0 + +if test "$debug" -eq 1 ; then + $bin/btape -w $tmp -c $conf/bacula-sd.conf LTO3_0 <$tmp/log1.out 2>&1 +test +y +quit +END_OF_DATA +fi + + +if [ $? != 0 ] ; then + echo " " + echo " " + echo " !!!!! btape test failed!!! !!!!! " + echo " !!!!! btape test failed!!! !!!!! " >>test.out + echo " " + exit 1 +fi + +grep "The test autochanger worked" $tmp/log1.out > /dev/null +if [ $? != 0 ] ; then + print_debug "btape autochanger test failed" + estat=1 +fi + +grep "End Forward space files test" $tmp/log1.out > /dev/null +if [ $? != 0 ] ; then + print_debug "btape test failed" + estat=1 +fi + +# now try to use the fill command +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumVolumeSize', '400M', 'Device')" + +$bin/disk-changer $working/ach load 1 $working/ach/drive0 0 + +if test "$debug" -eq 1 ; then + $bin/btape -w $tmp -c $conf/bacula-sd.conf LTO3_0 <$tmp/log2.out 2>&1 +fill +m +quit +END_OF_DATA +fi + + +if [ $? != 0 ] ; then + echo " " + echo " " + echo " !!!!! btape test failed!!! !!!!! " + echo " !!!!! btape test failed!!! !!!!! " >>test.out + echo " " + exit 1 +fi + +grep "The last block of the first tape matches." $tmp/log2.out > /dev/null +if [ $? != 0 ] ; then + print_debug "Could not find: The last block of the first tape matches." + print_debug "btape fill test failed" + estat=1 +fi + +grep "The first block on the second tape matches." $tmp/log2.out > /dev/null +if [ $? != 0 ] ; then + print_debug "Could not find: The first block on the second tape matches." + print_debug "btape fill test failed" + estat=1 +fi + +end_test diff --git a/regress/tests/btape-test-changer b/regress/tests/btape-test-changer index 435a82d901..4980bc08c0 100755 --- a/regress/tests/btape-test-changer +++ b/regress/tests/btape-test-changer @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Test the test command in btape # @@ -22,7 +20,7 @@ change_jobname $JobName start_test if test "$debug" -eq 1 ; then - $bin/btape -c bin/bacula-sd.conf Drive-0 <${cwd}/bin/bacula-sd.conf if test "$debug" -eq 1 ; then - $bin/btape -c bin/bacula-sd.conf Drive-0 <${cwd}/tmp/log2.out 2>&1 + $bin/btape -w $tmp -c bin/bacula-sd.conf Drive-0 <${cwd}/tmp/log2.out 2>&1 test yes quit diff --git a/regress/tests/cancel-inactive-test b/regress/tests/cancel-inactive-test new file mode 100755 index 0000000000..ac025ab46b --- /dev/null +++ b/regress/tests/cancel-inactive-test @@ -0,0 +1,68 @@ +#!/usr/bin/perl -w +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Small test to setup a backup with $nb_files*2 to backup, +# then, we can run multiple full jobs over the directory +# and create a "large" catalog. +# +use strict; + +# use bacula functions +use scripts::functions; + +start_test(); + +# cleanup the previous conf +cleanup(); + +# initialize the configuration +system("scripts//copy-test-confs"); + +# initialize the fileset +add_to_backup_list("$cwd/build/po"); + +start_bacula(); + +create_bconcmds( + "$out $tmp/log1.out", + "label volume=TestVolume001 storage=File pool=Scratch", + "run job=NightlySave level=full yes", + "wait", + "messages", + "$out $tmp/log2.out", + "llist jobid=1", + ); + +run_bconsole(); + +my $job = `awk '/[jJ]ob: / { print \$2 }' $tmp/log2.out`; +chomp($job); + +p("Now try to cancel the previous job to see if the director will connect to FD/SD"); + +create_bconcmds( + "$out $tmp/log3.out", + "cancel inactive client=$CLIENT ujobid=$job", + "$out $tmp/log4.out", + "cancel inactive ujobid=$job", + "$out $tmp/log5.out", + "cancel inactive storage=File ujobid=$job" + ); + +run_bconsole(); + +stop_bacula(); + +p("Should find FD code 2901 in cancel output"); + +if (!docmd("grep 2901 $tmp/log3.out") || + !docmd("grep 2901 $tmp/log4.out") || + !docmd("grep 2901 $tmp/log5.out")) +{ + print "ERROR: Unable to find 2901 in log3,4,5.out\n"; + $estat=1; +} + +end_test(); diff --git a/regress/tests/cancel-test b/regress/tests/cancel-test index 106e0ddf22..bc93a24697 100755 --- a/regress/tests/cancel-test +++ b/regress/tests/cancel-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Test different cancel cases # Can be quite long @@ -25,7 +23,11 @@ echo "s% Schedule =%# Schedule =%g" >$outf cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 sed -f ${outf} ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf -MAXVOLBYTES=1MB +if [ "$FORCE_DEDUP" = yes ]; then + MAXVOLBYTES=5KB +else + MAXVOLBYTES=1MB +fi touch tmp/log1.out @@ -85,7 +87,7 @@ when=`perl -MPOSIX -e "print strftime('%F %T', localtime(time+6000))"` cat <${cwd}/tmp/bconcmds @output /dev/null messages -setbandwidth limit=500 client +setbandwidth limit="500 kb/s" client @$out ${cwd}/tmp/logfile1.out @################################################################ @# run a first test without volume diff --git a/regress/tests/console-dotcmd-test b/regress/tests/console-dotcmd-test new file mode 100755 index 0000000000..98f2238cd1 --- /dev/null +++ b/regress/tests/console-dotcmd-test @@ -0,0 +1,113 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a couple of tests with bconsole and dotcommands +# +TestName="console-dotcmd-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-test-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +echo "$cwd/build/po" > $tmp/file-list + +change_jobname NightlySave $JobName + +perl -Mscripts::functions \ + -e "extract_resource('$conf/bacula-dir.conf', 'Client', '$HOST-fd')" \ + > $tmp/1 + +OUT=$conf/bacula-dir.conf +for i in `seq 0 254` +do + + sed "s/$HOST-fd/test$i-1-fd/" $tmp/1 | sed "s/Address.*/Address = 127.0.0.$i/" >> $OUT + sed "s/$HOST-fd/test$i-2-fd/" $tmp/1 | sed "s/Address.*/Address = 127.0.2.$i/" >> $OUT + sed "s/$HOST-fd/test$i-rst-fd/" $tmp/1 | sed "s/Address.*/Address = 127.0.3.$i/" >> $OUT + sed "s/$HOST-fd/test$i-err-fd/" $tmp/1 | sed "s/Address.*/Address = something.lan.xx.baculasystems.com/" >> $tmp/bad-fd +done + +start_test + +# We create hundred of clients in this test, the director startup can be long +# specially on MySQL and Ubuntu +touch $tmp/bconcmds +run_bacula + +ok=1 +retry=0 + +while [ $ok -ne 0 -a $retry -lt 5 ]; do + echo quit | $bin/bconsole -c $conf/bconsole.conf | grep quit + ok=$? + retry=`expr $retry + 1` +done + +cat < $tmp/bconcmds +@out /dev/null +reload +messages +@$out $tmp/log1.out +.client +@$out $tmp/log2.out +time +.client address=127.0.0.50 +time +@$out $tmp/log3.out +time +.client address=x.x.x.x +time +@echo done +@exec "touch $tmp/log.done" +EOF + +run_bconsole + +# Put some bad FDs in the list +cat $tmp/bad-fd >> $conf/bacula-dir.conf +sed s/log/logerr/ $tmp/bconcmds > $tmp/bconcmds.err + +# With DNS errors, it will take more times +(run_bconsole $tmp/bconcmds.err)& + +sleep 2 + +cat < $tmp/bconcmds.sametime +@out /dev/null +reload +messages +@$out $tmp/log1.out +.client +@exec "touch $tmp/logsametime.done" +EOF + +run_bconsole $tmp/bconcmds.sametime + +wait + +$bperl -e '(-M "$tmp/logsametime.done" < -M "$tmp/logerr.done") && exit 1' +if [ $? -ne 0 ]; then + print_debug "ERROR: The timestamp of $tmp/logsametime.done should be smaller than $tmp/logerr.done" + estat=1 +fi + +nb=`grep test50-1-fd $tmp/log2.out $tmp/logerr2.out | wc -l` +if [ $nb != 2 ]; then + print_debug "ERROR: Should find the client test50-1-fd in the $tmp/log2.out and $tmp/logerr2.out files" + estat=1 +fi + +stop_bacula +end_test diff --git a/regress/tests/copy-job-test b/regress/tests/copy-job-test index 49b2783895..14a2b68b0d 100755 --- a/regress/tests/copy-job-test +++ b/regress/tests/copy-job-test @@ -1,9 +1,4 @@ #!/bin/sh -# -# Copyright (C) 2000-2015 Kern Sibbald -# License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a simple backup of the Bacula build directory then copy it # to another device. @@ -17,7 +12,6 @@ JobName=CopyJobSave scripts/cleanup scripts/copy-migration-confs -scripts/prepare-disk-changer echo "${cwd}/build" >${cwd}/tmp/file-list sed 's/migrate/copy/g' ${cwd}/bin/bacula-dir.conf > ${cwd}/tmp/1 sed 's/Migrate/Copy/g' ${cwd}/tmp/1 > ${cwd}/bin/bacula-dir.conf diff --git a/regress/tests/copy-swap-fail-test b/regress/tests/copy-swap-fail-test new file mode 100755 index 0000000000..86b184d421 --- /dev/null +++ b/regress/tests/copy-swap-fail-test @@ -0,0 +1,159 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run backups with dummy tape driver +# This test setups an Autochanger with 80 slots +# and 5 drives (5 LTO3) +# +# + +TestName="copy-swap-fail-test" +JobName=backup +. scripts/functions + +require_vtape + +scripts/cleanup +scripts/copy-tape-confs +cp $rscripts/bacula-dir-vtape.conf $conf/bacula-dir.conf +sed 's/LTO1/LTO3/g' $rscripts/bacula-sd-vtape.conf > $conf/bacula-sd.conf +scripts/prepare-fake-autochanger + +echo "${cwd}/build" >${cwd}/tmp/file-list + +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Maximum Concurrent Jobs", "1", "Device")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Maximum Volume Size", "100MB", "Device")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "LabelMedia", "Yes", "Device", "FileStorage")' + +$bperl -e 'add_attribute("$conf/bacula-fd.conf", "MaximumBandwidthPerJob", "1200Kb/s", "FileDaemon")' + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Label Format", "TestVolume", "Pool", "Full")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Maximum Volume Jobs", "1", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "SpoolData", "no", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "NextPool", "Full", "Pool", "Inc")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Storage", "File", "Pool", "Full")' + +cat <> $conf/bacula-dir.conf +Job { + Name = "copy-job" + Type = Copy + Level = Full + Client=${HOST}-fd + FileSet="Full Set" + Messages = Standard + Storage = LTO3 + Write Bootstrap = "$working/migratejob.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = Job + Selection Pattern = ".*Save.*" +# Allow Duplicate Jobs = No +} +EOF + +perl -Mscripts::functions -e "extract_resource('$conf/bacula-dir.conf', 'Job', 'NightlySave')" > $tmp/1 +cat $tmp/1 | sed s/NightlySave/NightlySave4/ >> $conf/bacula-dir.conf +cat $tmp/1 | sed s/NightlySave/NightlySave5/ >> $conf/bacula-dir.conf +cat $tmp/1 | sed s/NightlySave/NightlySave6/ >> $conf/bacula-dir.conf +cat $tmp/1 | sed s/NightlySave/NightlySave7/ >> $conf/bacula-dir.conf +cat $tmp/1 | sed s/NightlySave/NightlySave8/ >> $conf/bacula-dir.conf +cat $tmp/1 | sed s/NightlySave/NightlySave9/ >> $conf/bacula-dir.conf +cat $tmp/1 | sed s/NightlySave/NightlySave10/ >> $conf/bacula-dir.conf + +start_test + +clientname=`awk '/Name = .*-fd/ { if (!ok) { print $3 ; ok=1 } }' bin/bacula-dir.conf` + +when1=`perl -MPOSIX -e "print strftime('%F %T', localtime(time+20))"` +when2=`perl -MPOSIX -e "print strftime('%F %T', localtime(time+35))"` +when3=`perl -MPOSIX -e "print strftime('%F %T', localtime(time+50))"` + +# Catalog record for cleaning tape "CLN01" successfully created. +# CLN01 | Cleaning + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log6.out +@#setdebug level=200 storage=LTO1 +label barcodes pool=Scratch slots=1-10 storage=LTO3 drive=0 +yes +messages +list volumes +END_OF_DATA + +run_bacula + +cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log1.out +setdebug trace=1 level=200 storage=LTO3 +mount storage=LTO3_2 slot=60 drive=2 +run storage=LTO3 when="$when1" job=NightlySave pool=Inc yes +run storage=LTO3 when="$when2" job=NightlySave2 pool=Inc yes +run storage=LTO3 when="$when3" job=NightlySave3 pool=Inc yes +wait jobid=1 +status storage=LTO3 +messages +run storage=LTO3_2 job=copy-job jobid=1 pool=Inc yes +wait jobid=2 +messages +run storage=LTO3_2 job=copy-job jobid=2 pool=Inc yes +wait jobid=3 +messages +run storage=LTO3_2 job=copy-job jobid=3 pool=Inc yes +wait +messages +quit +END_OF_DATA + +run_bconsole +check_for_zombie_jobs storage=LTO3 client=$clientname + +cat <${cwd}/tmp/bconcmds +setdebug trace=1 level=0 storage=LTO3 +@$out ${cwd}/tmp/log2.out +@# +@# now do a restore +@# +restore client=$clientname fileset="Full Set" where=${cwd}/tmp/bacula-restores select all done +yes +wait +messages +END_OF_DATA + +run_bconsole +check_for_zombie_jobs storage=LTO3 client=$clientname + +stop_bacula + +check_two_logs +check_restore_diff +perl -ne ' +# foreach Job, we store the drive and we count the total +# job number per drive +m/(\d+): Using Device "(.+?)"/ + and $drive_info[$i++]="$1 $2" and $drive{$2}++; + +END { + $err=0; + foreach $k (sort keys %drive) { + if ($drive{$k} > 3) { + print "ERR $k has more than 3 jobs ($drive{$k})\n"; + $err++; + } + } + if ($err) { + foreach $k (@drive_info) { print "$k\n"} + exit 1; + } +} +' $tmp/log1.out + +if [ $? -ne 0 ]; then + print_debug "ERR: Problem during log analysis" + estat=1 +fi +end_test diff --git a/regress/tests/copy-uncopied-test b/regress/tests/copy-uncopied-test index 10dd619a6c..29c98cc4d8 100755 --- a/regress/tests/copy-uncopied-test +++ b/regress/tests/copy-uncopied-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a backup of the Bacula build directory on two Volumes # then copy it to another device. @@ -17,7 +15,6 @@ JobName=MigVolBackup scripts/cleanup scripts/copy-migration-confs -scripts/prepare-disk-changer echo "${cwd}/build" >${cwd}/tmp/file-list #cp ${cwd}/bin/bacula-sd.conf ${cwd}/tmp/1 #sed "s%# Maximum File Size% Maximum File Size%" ${cwd}/tmp/1 >${cwd}/bin/bacula-sd.conf diff --git a/regress/tests/copy-volume-test b/regress/tests/copy-volume-test index d82ede8f41..9ddcf1a032 100755 --- a/regress/tests/copy-volume-test +++ b/regress/tests/copy-volume-test @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS # @@ -41,6 +41,7 @@ cat <${cwd}/tmp/bconcmds messages @$out ${cwd}/tmp/log1.out setdebug level=50 storage=DiskChanger +@#setdebug level=100 Dir label storage=File volume=FileVolume001 Pool=Default label storage=File volume=FileVolume002 Pool=Scratch update Volume=FileVolume001 MaxVolBytes=3000000 pool=Default diff --git a/regress/tests/crazy-smaller-vol-test b/regress/tests/crazy-smaller-vol-test new file mode 100755 index 0000000000..7263b885b2 --- /dev/null +++ b/regress/tests/crazy-smaller-vol-test @@ -0,0 +1,106 @@ +#!/bin/sh +# +# A shell and stripped down version version of the crazy-small-volume-test.py +# +# +TestName="crazy-smaller-vol-test" +JobName=Simple +. scripts/functions + +${rscripts}/cleanup +${rscripts}/copy-test-confs +mkdir -p ${tmpsrc} +dd if=/dev/urandom of=${tmpsrc}/the_file bs=1M count=20 +the_file_md5=`md5sum ${tmpsrc}/the_file | cut -d " " -f 1` +echo $the_file_md5 +cd ${cwd} + +echo ${tmpsrc}/the_file>${cwd}/tmp/file-list + +$bperl -e "set_global_maximum_concurrent_jobs(10)" + +start_test + +numvols=220 +numjobs=20 +maxbytes=2097152 + +cat <${tmp}/bconcmds +@output /dev/null +messages +@output /dev/null +setdebug level=15 storage=File +@#setdebug level=200 client +@#setdebug level=100 director + +END_OF_DATA + +i=0 +while [ $i -lt $numvols ] ; do + cat <>${tmp}/bconcmds +label storage=File volume=TestVolume$i +update volume=TestVolume$i MaxVolBytes=${maxbytes} +END_OF_DATA + i=$(($i + 1)) +done + +# Make one more big volume +cat <>${tmp}/bconcmds +label storage=File volume=TestVolume1000 +@# start logging +@$out ${tmp}/log1.out +END_OF_DATA + +i=0 +while [ $i -lt ${numjobs} ] ; do + cat <>${tmp}/bconcmds +run job=Simple yes +END_OF_DATA + i=$(($i + 1)) +done + +cat <>${tmp}/bconcmds +wait +messages +END_OF_DATA + +cat ${tmp}/bconcmds + +run_bacula +check_for_zombie_jobs storage=File + + +my_error=0 +i=0 +while [ $i -lt ${numjobs} ] ; do + rm -rf ${tmp}/bacula-restores + mkdir -p ${tmp}/bacula-restores + i=$(($i + 1)) + cat <${tmp}/bconcmds +@output /dev/null +messages +@$out ${tmp}/log2.out +list joblog jobid=%i +sql "select * from JobMedia WHERE JobId=%i;" + +restore jobid=$i where=${tmp}/bacula-restores all storage=File done yes +wait +messages +END_OF_DATA + run_bconsole + ls -l ${tmp}/bacula-restores/${tmpsrc}/the_file + md5=`md5sum ${tmp}/bacula-restores/${tmpsrc}/the_file | cut -d " " -f 1` + if [ $the_file_md5 != $md5 ] ; then + echo ERROR ERROR ERROR ERROR ERROR job $i + my_error=1 + fi +done + + +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +end_test + +exit $my_error diff --git a/regress/tests/four-drive-test b/regress/tests/four-drive-test new file mode 100755 index 0000000000..c5cb5aa27b --- /dev/null +++ b/regress/tests/four-drive-test @@ -0,0 +1,88 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# This script is used to test multiple devices grouped in +# the director configuration +# +# +TestName="four-drive-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-4disk-confs +scripts/prepare-disk-changer + +echo "s/signature=MD5/signature=MD5; readfifo=yes/" > $tmp/1 +echo "s/FileStorage/FileStorage; Device=FileStorage2; Device=FileStorage3; Device=FileStorage4/" >> $tmp/1 +sed -f $tmp/1 $conf/bacula-dir.conf > $tmp/2 +# Allow auto label +$bperl -e 'add_attribute("$tmp/2", "Label Format", "Vol", "Pool", "Default")' +$bperl -e 'add_attribute("$tmp/2", "Label Format", "Vol", "Pool", "Inc")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Label Media", "yes", "Device")' + +# set this to do round robin +$bperl -e 'set_maximum_concurrent_jobs("$conf/bacula-sd.conf", 8, "Device")' + +# Disable spooling for each job +$bperl -e 'add_attribute("$tmp/2", "SpoolData", "no", "Job")' +cp $tmp/2 $conf/bacula-dir.conf + +$bperl -e 'extract_resource("$conf/bacula-sd.conf", "Device", "FileStorage")' > $tmp/2 +for i in 2 3 4; do + sed "s/FileStorage/FileStorage$i/" $tmp/2 >> $conf/bacula-sd.conf +done + +disable_plugins + +echo "$cwd/build/src/dird/" >${cwd}/tmp/file-list + +change_jobname $JobName +start_test + +# test with autolabel +# +# +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@out /dev/null +messages +@#setdebug level=150 storage=Virtual +@$out ${cwd}/tmp/log1.out +run level=full job=backup storage=Virtual pool=Default yes +run level=full job=backup storage=Virtual pool=Default yes +run level=full job=backup storage=Virtual pool=Default yes +run level=full job=backup storage=Virtual pool=Default yes +run level=full job=backup storage=Virtual pool=Inc yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Default yes +@#run level=full job=backup storage=Virtual pool=Inc yes +@#run level=full job=backup storage=Virtual pool=Inc yes +@sleep 3 +status dir +messages +wait +messages +status storage=Virtual +quit +END_OF_DATA + +run_bacula + +check_for_zombie_jobs storage=File +check_for_zombie_jobs storage=tape +stop_bacula + +touch $tmp/log2.out +check_two_logs +#check_restore_diff + +end_test diff --git a/regress/tests/four-jobs-tape b/regress/tests/four-jobs-tape index 924518e06b..e1747c513b 100755 --- a/regress/tests/four-jobs-tape +++ b/regress/tests/four-jobs-tape @@ -38,7 +38,6 @@ END_OF_DATA run_bacula scripts/check_for_zombie_jobs storage=tape -echo "Backup 1 done" # make some files for the incremental to pick up touch ${cwd}/build/src/dird/*.c ${cwd}/build/src/dird/*.o touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o @@ -59,7 +58,6 @@ END_OF_DATA run_bacula scripts/check_for_zombie_jobs storage=tape -echo "Backup 2 done" touch ${cwd}/build/src/dird/*.c touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o # @@ -78,7 +76,6 @@ END_OF_DATA run_bacula scripts/check_for_zombie_jobs storage=tape -echo "Backup 3 done" # make some files for the incremental to pick up touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o # @@ -96,7 +93,6 @@ END_OF_DATA run_bacula scripts/check_for_zombie_jobs storage=tape -echo "Backup 4 done" # # now do several restores to ensure we cleanup between jobs # diff --git a/regress/tests/incremental-2media b/regress/tests/incremental-2media index 9417c7d079..a2d162d0e4 100755 --- a/regress/tests/incremental-2media +++ b/regress/tests/incremental-2media @@ -39,7 +39,6 @@ END_OF_DATA run_bacula -echo "Backup 1 done" # make some files for the incremental to pick up touch ${cwd}/build/src/dird/*.c ${cwd}/build/src/dird/*.o touch ${cwd}/build/src/lib/*.c ${cwd}/build/src/lib/*.o diff --git a/regress/tests/incremental-changer b/regress/tests/incremental-changer index 8d4a5c6a7c..f0f38dac60 100755 --- a/regress/tests/incremental-changer +++ b/regress/tests/incremental-changer @@ -21,6 +21,8 @@ scripts/cleanup scripts/copy-2tape-confs scripts/prepare-two-tapes +exit + echo "${cwd}/tmp/build" >${cwd}/tmp/file-list if test ! -d ${cwd}/tmp/build ; then mkdir -p ${cwd}/tmp/build diff --git a/regress/tests/jobmedia-bug-test b/regress/tests/jobmedia-bug-test new file mode 100755 index 0000000000..3b1f45f699 --- /dev/null +++ b/regress/tests/jobmedia-bug-test @@ -0,0 +1,117 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# By setting VolUse duration small and switching Volume +# we create bad JobMedia records. EndBlock < StartBlock. +# +# +TestName="jobmedia-bug-test" +JobName=jobmedia-bug +. scripts/functions + +${rscripts}/cleanup +${rscripts}/copy-test-confs +echo $tmp/big >${tmp}/file-list +echo $cwd/build/po >>${tmp}/file-list + +dd if=/dev/zero of=$tmp/big seek=1000000 count=1 >/dev/null 2>&1 +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "SpoolData", "No", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "ClientRunBeforeJob", "\"sleep 30\"", "Job", "NightlySave")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumConcurrentJobs", "10", "Device")' +sed -i 's/sparse=yes;//' $conf/bacula-dir.conf + +change_jobname NightlySave $JobName +start_test + +cat <${tmp}/bconcmds +@output /dev/null +messages +@$out ${tmp}/log1.out +label storage=File volume=TestVolume000 +label storage=File volume=TestVolume001 +label storage=File volume=TestVolume002 +@#setdebug level=150 storage +@# With VolUse=15s nothing restored for JobId 2 +@#update volume=TestVolume000 VolUse=15s +@# With VolUse=20s both JobId 2 and 3 restore wrong size +update volume=TestVolume000 VolUse=20s +update volume=TestVolume001 maxvolbytes=100MB +sql +select VolumeName, MediaId FROM Media; + +run job=$JobName level=full yes +@sleep 15 +setbandwidth limit="5000 kb/s" client=$CLIENT +run job=$JobName level=full yes +run job=$JobName level=full yes +@sleep 20 +run job=$JobName level=full yes +@sleep 10 +@#status client +@#status dir +@#status storage +@sleep 10 +setbandwidth limit=0 client=$CLIENT +wait +messages +sql +select jobid, mediaid, firstindex,lastindex,volindex from JobMedia order by jobid,volindex; +select * from JobMedia order by jobid,volindex; + +@$out $tmp/jobmedia2 +list jobmedia jobid=2 +@$out $tmp/jobmedia3 +list jobmedia jobid=3 +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File + +cat <${tmp}/bconcmds +@# +@# now do a restore that will fail (JobId 2) +@# +@$out ${tmp}/log2.out +@#setdebug level=10 storage=File +@# Select by JobId to restore JobId 1 +restore where=${tmp}/bacula-restores storage=File +3 +1 +cd $tmp +m big +done +yes +wait +@# Select by JobId to restore JobId 2 +restore where=${tmp}/bacula-restores storage=File +3 +2 +cd $tmp +m big +done +yes +wait +@# Select by JobId to restore JobId 3 +restore where=${tmp}/bacula-restores2 storage=File +3 +3 +cd $tmp +m big +done +yes +wait +@#status client +@#status storage=File +messages +quit +END_OF_DATA + +run_bconsole +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +end_test diff --git a/regress/tests/maxbw-test b/regress/tests/maxbw-test index 87ecc7961d..81daeab8ae 100755 --- a/regress/tests/maxbw-test +++ b/regress/tests/maxbw-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Verify Maximum Bandwidth FD parameter # @@ -12,6 +10,11 @@ TestName="maxbw-test" JobName=NightlySave . scripts/functions +# +# This test does not work +# +exit 0 + scripts/cleanup scripts/copy-test-confs diff --git a/regress/tests/maxbytes-test b/regress/tests/maxbytes-test index b3c1a7555e..8fa54db09e 100755 --- a/regress/tests/maxbytes-test +++ b/regress/tests/maxbytes-test @@ -46,8 +46,10 @@ mark * done yes wait +reload @#reload -@#reload +status client +cancel inactive jobid=1,2 messages quit END_OF_DATA diff --git a/regress/tests/maxuseduration-test b/regress/tests/maxuseduration-test index 5cde51a0aa..377f1679d7 100755 --- a/regress/tests/maxuseduration-test +++ b/regress/tests/maxuseduration-test @@ -1,9 +1,4 @@ #!/bin/sh -# -# Copyright (C) 2000-2015 Kern Sibbald -# License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run four jobs at the same time, with four Volumes, but set # Maximum Job Volumes = 1 on each of the Volumes. Note, @@ -31,7 +26,7 @@ update Volume=TestVolume001 20 1 3 -18 +19 llist volume=TestVolume001 @#setdebug level=100 Storage=File1 run job=$JobName level=Full Storage=File1 yes @@ -39,7 +34,7 @@ run job=$JobName level=Full Storage=File1 yes update Volume=TestVolume001 1 1 -18 +19 mount storage=File1 wait llist volume=TestVolume001 diff --git a/regress/tests/mcj-test b/regress/tests/mcj-test new file mode 100755 index 0000000000..e92a335e55 --- /dev/null +++ b/regress/tests/mcj-test @@ -0,0 +1,78 @@ +#!/bin/sh +# +# Run a simple backup of the Bacula build directory and see +# if bacula is respecting the maximum concurrent job (MJC) +# +TestName="mjc-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +change_jobname BackupClient1 $JobName +start_test + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +show client +setbandwidth limit="100 kb/s" client +run job=$JobName level=full yes +@sleep 5 +status dir +@sleep 5 +run job=$JobName level=full comment="Should be waiting for Maximum Concurrent Job" yes +@sleep 5 +status dir +@# The following reload command is causing a bug with the current code +reload +@sleep 2 +run job=$JobName level=full comment="Should be waiting for Maximum Concurrent Job" yes +@sleep 5 +@$out $tmp/log3.out +@# ################################################################################## +@# If everything is alright, we should find two jobs waiting for MaximumConcurrentJob +@# ################################################################################## +status dir +@# ################################################################################## +@$out $tmp/log1.out +setbandwidth limit="10000000 kb/s" client +wait +messages +@# +@# now do a restore +@# +@$out $tmp/log2.out +restore where=$tmp/bacula-restores select all done +yes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File1 +stop_bacula + +check_two_logs +check_restore_diff + +nb=$(grep "is waiting" $tmp/log3.out | wc -l) +if [ $nb -ne 2 ]; then + print_debug "ERROR: Should find two jobs in $tmp/log3.out waiting for Maximum Concurrent Jobs, got $nb" + estat=1 +fi + +end_test diff --git a/regress/tests/messages-saved-test b/regress/tests/messages-saved-test new file mode 100755 index 0000000000..a439897f57 --- /dev/null +++ b/regress/tests/messages-saved-test @@ -0,0 +1,59 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a simple backup of the Bacula build directory +# See if we have the list of files backed up in a log +# +# +TestName="messages-saved-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs +change_jobname BackupClient1 $JobName +start_test + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "ClientRunBeforeJob", "echo this is a test", "Job", "backup")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "ClientRunAfterJob", "echo this is a test", "Job", "backup")' +$bperl -e 'add_attribute("$conf/bacula-fd.conf", "Append", "$tmp/fd.log = All, Saved", "Messages", "Standard")' + +cat <$tmp/bconcmds +@output /dev/null +messages +@$out $tmp/log1.out +@#setdebug level=100 storage=File +label volume=TestVolume001 storage=File1 pool=File drive=0 slot=0 +run job=$JobName yes +@sleep 1 +status storage=File1 +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +nb=`cat $tmp/fd.log | wc -l` +if [ $nb -lt 1000 ]; then + print_debug "ERROR: Not enough line in $tmp/fd.log" + estat=1 +fi + +grep po/fr.po $tmp/fd.log > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: Should find po/fr.po in $tmp/fd.log" + estat=1 +fi + +grep "ClientRunBeforeJob: this is a test" $tmp/fd.log > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: Should runscript output in $tmp/fd.log" + estat=1 +fi + +end_test diff --git a/regress/tests/multi-storage-test b/regress/tests/multi-storage-test index f7d3982a24..5c915ca21b 100755 --- a/regress/tests/multi-storage-test +++ b/regress/tests/multi-storage-test @@ -26,7 +26,7 @@ print; mkdir -p ${working}2 perl -ne ' -if (/^Storage {/) { $in=1; $nb++; } +if (/^Storage \{/) { $in=1; $nb++; } if (/^}/) { $in=0 } if (/SDPort = (\d+)/ && $in) {if ($nb == 2) { $_ = " SDPort = 1$1\n"; }} print; diff --git a/regress/tests/next-pool-test b/regress/tests/next-pool-test new file mode 100755 index 0000000000..cbdc15611a --- /dev/null +++ b/regress/tests/next-pool-test @@ -0,0 +1,200 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test various nextpool options for copy/migration +# +TestName="next-pool-test" +JobName=Virtual + +. scripts/functions +${rscripts}/cleanup +cp -f ${rscripts}/bacula-dir-2media-virtual.conf ${conf}/bacula-dir.conf +cp -f ${rscripts}/bacula-sd-2media-virtual.conf ${conf}/bacula-sd.conf +cp -f ${rscripts}/test-bacula-fd.conf ${conf}/bacula-fd.conf +cp -f ${rscripts}/test-console.conf ${conf}/bconsole.conf + +echo "$cwd/build/src/dird" >${cwd}/tmp/file-list + +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Label Media", "Yes", "Device")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Label Format", "Vol-", "Pool")' + +# create where the SD is putting volumes +mkdir -p $tmp/disk + +# The 3rd drive doesn't have the right type +sed 's/disk1/disk/' $conf/bacula-sd.conf > 1 +cp -f 1 $conf/bacula-sd.conf + +start_test + +cat <${tmp}/bconcmds +@output /dev/null +messages +@$out $tmp/log1.out +run job=Virtual yes +wait +messages +@$out $tmp/log3.out +@# Looking for: FullCopy +run job=copy-job jobid=1 +yes +wait +messages +@$out $tmp/log4.out +@# Looking for: Special +run job=copy-job jobid=1 +mod +9 +4 +yes +wait +messages +@$out $tmp/log5.out +@# Looking for: Special +run job=copy-job jobid=1 nextpool=Special +yes +wait +messages +@$out $tmp/log6.out +run job=copy-job-next-pool jobid=1 +yes +wait +messages +@$out $tmp/log7.out +run job=copy-job-next-pool nextpool=FullCopy jobid=1 +yes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=Virtual +stop_bacula + +# menu +# Pool: Default (From Job resource) +# NextPool: FullCopy (From Job Pool's NextPool resource) +# Read Storage: Virtual (From Job resource) +# Write Storage: Virtual (From Job Pool's NextPool resource) + +# output +# Read Pool: "Default" (From Job resource) +# Read Storage: "Virtual" (From Job resource) +# Write Pool: "FullCopy" (From Job Pool's NextPool resource) +# Write Storage: "Virtual" (From Job Pool's NextPool resource) + +grep "NextPool: FullCopy (From Job Pool's NextPool resource)" $tmp/log3.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log3.out, should be FullCopy" + estat=1 +fi + +grep "Write Storage: Virtual (From Job Pool" $tmp/log3.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad Write Storage in $tmp/log3.out, should be Virtual" + estat=1 +fi + +grep 'Write Pool: "FullCopy"' $tmp/log3.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad Write Pool in $tmp/log3.out, should be FullCopy" + estat=1 +fi + +# log4.out +# Pool: Default (From Job resource) +# NextPool: Special (From User input) +# Read Storage: Virtual (From Job resource) +# Write Storage: Virtual (From User input) + +# Read Pool: "Default" (From Job resource) +# Read Storage: "Virtual" (From Job resource) +# Write Pool: "Special" (From User input) +# Write Storage: "Virtual" (From User input) + +grep "NextPool: Special (From User input)" $tmp/log4.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log4.out, should find Special" + estat=1 +fi + +grep "Write Storage: Virtual2 (From User input)" $tmp/log4.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad Write Storage in $tmp/log4.out, should be Virtual" + estat=1 +fi + +grep 'Write Pool: "Special"' $tmp/log4.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad Write Pool in $tmp/log4.out, should be Special" + estat=1 +fi + +# log5.out +# Pool: Default (From Job resource) +# NextPool: Special (From Command input) +# Read Storage: Virtual (From Job resource) +# Write Storage: Virtual2 (From Command input) + +# Read Pool: "Default" (From Job resource) +# Read Storage: "Virtual" (From Job resource) +# Write Pool: "Special" (From Command input) +# Write Storage: "Virtual2" (From Command input) + +grep "NextPool: Special (From Command input)" $tmp/log5.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log5.out, should find Special" + estat=1 +fi + +grep "Write Storage: Virtual2 (From Command input)" $tmp/log5.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad Write Storage in $tmp/log5.out, should be Virtual" + estat=1 +fi + +grep 'Write Pool: "Special"' $tmp/log5.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad Write Pool in $tmp/log5.out, should be Special" + estat=1 +fi + + +# log6.out +# Pool: Default (From Job resource) +# NextPool: Special (From Job resource) +# Write Pool: "Special" (From Job resource) + +grep "NextPool: Special (From Job resource)" $tmp/log6.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log6.out, should find Special" + estat=1 +fi + +grep 'Write Pool: "Special" (From Job resource)' $tmp/log6.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log6.out, should find Special" + estat=1 +fi + +# log7.out +# NextPool: FullCopy (From command line) +# Write Pool: "FullCopy" (From command line) + +grep "NextPool: FullCopy (From Command input)" $tmp/log7.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log7.out, should find Special" + estat=1 +fi + +grep 'Write Pool: "FullCopy" (From Command input)' $tmp/log7.out > /dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Bad NextPool in $tmp/log7.out, should find Special" + estat=1 +fi + + +end_test diff --git a/regress/tests/next-vol-bug-7302 b/regress/tests/next-vol-bug-7302 new file mode 100755 index 0000000000..21d5fb675c --- /dev/null +++ b/regress/tests/next-vol-bug-7302 @@ -0,0 +1,119 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# This should expose bug #7302 +# +# This script uses the virtual disk autochanger and 30 drives +# +TestName="next-vol-bug-7302" +JobName="backup" +. scripts/functions + +scripts/cleanup +scripts/copy-2disk-tape-confs +CLIENT=2drive2disk + +echo "${cwd}/build" >${cwd}/tmp/file-list +change_jobname NightlySave $JobName +start_test + +# Turn on automatic label +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "LabelFormat", "vol", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "LabelMedia", "yes", "Device")' + +# Allow only one job per volume +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumVolumeJobs", "1", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumConcurrentJobs", "1", "Device")' + +# turn off spooling +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "SpoolData", "no", "Job")' + +# Create 30 drives for autochanger tape +for i in `seq 2 40`; do + $bperl -e 'extract_resource("$conf/bacula-sd.conf", "Device", "Drive-0")' | \ + sed "s/Drive-0/Drive-$i/" >> $conf/bacula-sd.conf +done + +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Device", join(",", map { "Drive-$_" } 0..30), "Autochanger", "tape")' + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@out /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug level=100 client=$CLIENT +setbandwidth client limit="2000 kb/s" +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +@sleep 2 +list volumes +messages +@sleep 20 +messages +@sleep 20 +messages +status storage=tape +status dir +list volumes +setbandwidth client limit="100000000 kb/s" +wait +messages +list volumes +quit +END_OF_DATA + +# exit + +run_bacula + +check_for_zombie_jobs storage=tape +stop_bacula + +touch $tmp/log2.out +check_two_logs + +end_test diff --git a/regress/tests/next-vol-test b/regress/tests/next-vol-test index f65e427270..2fdf74d150 100755 --- a/regress/tests/next-vol-test +++ b/regress/tests/next-vol-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # This script will test next vol algo with # vtape @@ -20,6 +18,17 @@ # . scripts/functions +# Dedup does not work with "tapes" +if test x$FORCE_DEDUP = xyes ; then + echo "\n=== Test next-vol-test skipped not compatible with dedup ===" + exit 0 +fi +if test x$FORCE_CLOUD = xyes ; then + echo "\n=== Test next-vol-test skipped not compatible with Cloud ===" + exit 0 +fi + + TestName="next-vol-test" JobName=backup diff --git a/regress/tests/poll-interval-test b/regress/tests/poll-interval-test new file mode 100755 index 0000000000..0584a6f9a9 --- /dev/null +++ b/regress/tests/poll-interval-test @@ -0,0 +1,79 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# + +# +# Check if polling interval interferes triggers exta mail +# commands. +# +# Note we use the viritual disk autochanger +# +TestName="poll-interval-test" +JobName=TwoVolume +. scripts/functions + +cwd=`pwd` +scripts/cleanup +scripts/copy-2disk-confs +scripts/prepare-disk-changer + +echo "${cwd}/build" >${cwd}/tmp/file-list + +outf="tmp/sed_tmp" +echo "s%# Maximum File Size% Maximum File Size%g" >${outf} +cp ${cwd}/bin/bacula-sd.conf ${cwd}/tmp/1 +sed -f ${outf} ${cwd}/tmp/1 >${cwd}/bin/bacula-sd.conf + +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "VolumePollInterval", 15, "Device")' + +change_jobname NightlySave $JobName +start_test + +# Write out bconsole commands +cat <tmp/bconcmds +@output /dev/null +messages +@$out tmp/log1.out +label storage=tape volume=TestVolume001 slot=1 pool=Default drive=0 +label storage=tape volume=TestVolume002 slot=2 pool=Default drive=0 +update Volume=TestVolume001 MaxVolBytes=3000000 pool=Default drive=0 +sql +select * from Storage; +select VolumeName,InChanger,slot,StorageId from Media; + +update volume=TestVolume002 slot=0 +setdebug level=50 storage=tape +list volumes +run job=$JobName yes +wait +messages +sql +select VolumeName,InChanger,slot,StorageId from Media; + +update slots scan storage=tape +messages +sql +select * from Storage; +select VolumeName,InChanger,slot,StorageId from Media; +select jobid,mediaid,startblock,endblock from JobMedia; + +@# +@# now do a restore +@# +@$out tmp/log2.out +restore where=${cwd}/tmp/bacula-restores select all storage=tape done +yes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=tape +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/pool-attributes-test b/regress/tests/pool-attributes-test new file mode 100755 index 0000000000..baf5ae6dbd --- /dev/null +++ b/regress/tests/pool-attributes-test @@ -0,0 +1,109 @@ +#!/bin/sh +# +# See if Pool/Media attributes are correctly handled +# +TestName="pool-attributes-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'CacheRetention', '10', 'Pool', 'File')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'CacheRetention', '20', 'Pool', 'Default')" + +change_jobname BackupClient1 $JobName +start_test + +cat <$tmp/bconcmds +@output /dev/null +messages +@$out $tmp/log1.out +setdebug level=4 storage=File1 +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +label volume=TestVolume002 storage=File1 pool=Default slot=1 drive=0 +@$out $tmp/01.log +llist volume=TestVolume001 +@$out $tmp/02.log +llist volume=TestVolume002 +quit +END_OF_DATA + +run_bacula + +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'CacheRetention', '30', 'Pool', 'File')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'CacheRetention', '40', 'Pool', 'Default')" +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'AutoLabel', 'yes', 'Device')" + + +cat <$tmp/bconcmds +@$out $tmp/log1.out +reload +update pool=File +update pool=Default +update allfrompool=File volume +update allfrompool=Default volume +@$out $tmp/11.log +llist volume=TestVolume001 +@$out $tmp/22.log +llist volume=TestVolume002 +@$out $tmp/log1.out +@# Check volumes from Scratch +delete volume=TestVolume001 yes +update volume=TestVolume002 pool=Scratch cacheretention=0 +run job=$JobName yes +wait +messages +@$out $tmp/31.log +llist volume=TestVolume001 +@$out $tmp/32.log +llist volume=TestVolume002 +@# Delete all volumes and check autolabel +@$out $tmp/log1.out +delete volume=TestVolume002 yes +run job=$JobName yes +wait +messages +@$out $tmp/41.log +llist volume +@$out $tmp/42.log +llist volume pool=File +quit +END_OF_DATA + +run_bconsole + +stop_bacula + +check_cacheretention() +{ + FILE=$1 + VAL=$2 + MSG=$3 + HAVE=`cat $FILE | tr 'A-Z' 'a-z' | awk '/cacheretention/ { print $2 }'` + if [ "$VAL" -ne "$HAVE" ]; then + print_debug "ERROR: Expect CacheRetention $VAL in $FILE, got $HAVE. $MSG" + estat=1 + fi +} + +check_cacheretention $tmp/01.log 10 "Check cacheretention with label command" +check_cacheretention $tmp/02.log 20 "Check cacheretention with label command" +check_cacheretention $tmp/11.log 30 "Check cacheretention with update frompool command" +check_cacheretention $tmp/22.log 40 "Check cacheretention with update frompool command" +check_cacheretention $tmp/32.log 30 "Check cacheretention that when we pull a volume from the scratch" +check_cacheretention $tmp/41.log 30 "Check cacheretention when we create a volume" +check_cacheretention $tmp/42.log 30 "Check cacheretention with pool= parameter" + + + +end_test diff --git a/regress/tests/priority-test b/regress/tests/priority-test new file mode 100755 index 0000000000..2e9beb11b5 --- /dev/null +++ b/regress/tests/priority-test @@ -0,0 +1,347 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a couple of jobs to test the priority feature +# +TestName="priority-test" +JobName=backup +. scripts/functions + +# Deactivate this test because:. +# 1. It runs too long -- more than 5 minutes +# 2. It detects errors, but that is not reported in +# the status. +# +# Please do not re-activate it until it is fixed: KES 03Feb17 +#exit 1 + +if [ x$FORCE_DEDUP = xyes ]; then + exit 0 +fi + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +change_jobname BackupClient1 $JobName +start_test + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumConcurrentJobs", 1, "Client")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumConcurrentJobs", 1000, "Director")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "AllowMixedPriority", "yes", "Job")' +cat <> $conf/bacula-dir.conf +FileSet { + Name = Small + Include { + File = $cwd/build/po/fr.po + } +} +EOF + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +setdebug level=4 storage=File1 +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +setbandwidth limit="10000 kb/s" client +run job=$JobName level=Full yes comment="1" priority=10 +run job=$JobName level=Full yes comment="X" priority=10 +run job=$JobName level=Full yes comment="2" priority=1 +run job=$JobName level=Full yes comment="X" priority=10 +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +run job=$JobName yes priority=10 fileset=Small +@sleep 5 +status dir +wait jobid=1 +@# The second job should be running now +run job=$JobName yes priority=2 comment="3" +setbandwidth limit="1000000 kb/s" client +wait +messages +@$out $tmp/log2.out +llist jobs limit=5 order=asc +quit +END_OF_DATA + +run_bacula +stop_bacula + +# Now, we check the job order, we should find the first job then +# the two jobs with the right priorities. +awk '/[Cc]omment:/ { print $2 }' $tmp/log2.out > $tmp/log3.out + +cat <> $tmp/log4.out +1 +2 +3 +X +X +EOF + +diff -Naur $tmp/log3.out $tmp/log4.out +if [ $? -ne 0 ]; then + print_debug "ERROR: The job order does not respect the priority that was set" +fi + +end_test diff --git a/regress/tests/prune-pool-test b/regress/tests/prune-pool-test new file mode 100755 index 0000000000..50acdfd4a9 --- /dev/null +++ b/regress/tests/prune-pool-test @@ -0,0 +1,151 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# This script will test prune algo +# +. scripts/functions + +TestName="prune-pool-test" +JobName=NightlySave + +scripts/cleanup +scripts/copy-test-confs + +start_test + +echo $PWD/build/po > tmp/file-list + +dircfg=$conf/bacula-dir.conf + +# copy the Default pool to Special pool +cat >> $dircfg < ${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out $tmp/log1.out +label storage=File pool=Default volume=TestVolume001 +label storage=File pool=Special volume=TestVolume002 +run job=$JobName pool=Default level=full yes +wait +messages +@exec "touch $cwd/build/po/fr.po" +run job=$JobName pool=Special level=incremental yes +wait +messages +@exec "touch $cwd/build/po/de.po" +run job=$JobName pool=Special level=incremental yes +wait +messages +@exec "touch $cwd/build/po/es.po" +run job=$JobName level=incremental yes +wait +messages +run job=$JobName pool=Default level=Differential yes +wait +messages +@sleep 2 +@$out $tmp/log3.out +@################################################################ +@# Should not prune anything +@#setdebug level=50 director +prune files pool=Special yes +list files jobid=2 +list files jobid=4 +list jobs +prune jobs pool=Special yes +list jobs +@################################################################ +@$out $tmp/log5.out +prune files pool=Default yes +list files jobid=4 +list jobs +prune jobs pool=Default yes +list jobs +@################################################################ +@$out $tmp/log4.out +@# Should prune incrementals +prune files yes +list files jobid=2 +list jobs +prune jobs yes +list jobs +@#setdebug level=0 director +@################################################################ +@$out $tmp/log2.out +restore where=${cwd}/tmp/bacula-restores select all storage=File done +yes +wait +messages +quit +EOF + +run_bacula + +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs + +estat=0 + +############################################################################### +# Now we will verify that the pruning code is working as expected. Each time, +# we run 'list jobs', 'prune', 'list jobs'. check_prune_list ensures that jobids +# in argument are present in the first 'list jobs', the 'prune' command deletes +# them (same number), and checks that the last 'list jobs' doesn't contain them +# anymore. See scripts/functions.pm for details. + +# nothing should be pruned +$bperl -e "check_prune_list('$tmp/log3.out')" +estat=$(($estat + $?)) + +# should prune only Default incremental +# F I I I D -> F I I D +$bperl -e "check_prune_list('$tmp/log5.out', 4)" +estat=$(($estat + $?)) + +# we should find fr.po in list files of jobid 2 +grep po/fr.po $tmp/log3.out > /dev/null +estat=$(($estat + $?)) + +# we should find es.po in list files of jobid 4 +grep po/es.po $tmp/log3.out > /dev/null +estat=$(($estat + $?)) + +# jobids 2 and 3 should be pruned +# (F I I I D) -> (F D) +$bperl -e "check_prune_list('$tmp/log4.out',2,3)" +estat=$(($estat + $?)) + +# we should not find fr.po in list files of jobid 2 +grep po/fr.po $tmp/log4.out > /dev/null +if [ $? = 0 ]; then + print_debug "ERROR: found fr.po in $tmp/log4.out after the prune file" + estat=$(($estat + 1)) +fi + +# we should not find es.po in list files of jobid 3 +grep po/es.po $tmp/log5.out > /dev/null +if [ $? = 0 ]; then + print_debug "ERROR: found es.po in $tmp/log5.out after the prune file" + estat=$(($estat + 1)) +fi + +end_test diff --git a/regress/tests/relabel-tape b/regress/tests/relabel-tape index f1a0a5a12d..39023ad082 100755 --- a/regress/tests/relabel-tape +++ b/regress/tests/relabel-tape @@ -61,7 +61,6 @@ run_bacula check_for_zombie_jobs storage=tape -echo "Backup done" # # now do several restores to ensure we cleanup between jobs # diff --git a/regress/tests/remote-console-test b/regress/tests/remote-console-test new file mode 100755 index 0000000000..ac6e8db911 --- /dev/null +++ b/regress/tests/remote-console-test @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test and setup the Client Initiated Backup +# +TestName="remote-console-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +change_jobname BackupClient1 $JobName +start_test + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Address", "0.0.0.0", "Client")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumBandwidthPerJob", "1mb/s", "Client")' +$bperl -e 'extract_resource("$conf/bconsole.conf", "Director")' | sed 's/Director/Console/' > $tmp/1 +cat $tmp/1 >> $conf/bacula-fd.conf +pass=`grep -i password $tmp/1` +name=`grep -i name $tmp/1` + +cat <> $conf/bacula-dir.conf +Console { + $name + $pass + CommandAcl = run, .api, restore, wait, status, .status, .jobs, .clients, .storages, .pools, .filesets, .defaults + jobacl = *all* + poolacl = *all* + clientacl = *all* + storageacl = *all* + catalogacl = *all* + filesetacl = *all* + directoryacl = *all* + useridacl = *all* + whereacl = *all* +} +EOF + +cat <> $conf/bacula-fd.conf +Director { + Name = remote-cons + Password = "oi3deith3peeGho4" + Remote = yes +} +EOF + +FDPORT=`expr $BASEPORT + 1` + +cat < $tmp/bconsole-remote.conf +Director { + Name = remote-cons + DIRport = $FDPORT + address = localhost + Password = "noused" +} + +Console { + Name = remote-cons + Director = remote-cons + Password = "oi3deith3peeGho4" +} +EOF + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +quit +END_OF_DATA + +run_bacula + +echo -e "proxy\n.status dir header api=2 api_opts=" | $bin/bconsole -c $tmp/bconsole-remote.conf +echo -e ".status header api=2 api_opts=" | $bin/bconsole -c $tmp/bconsole-remote.conf +echo -e "proxy\nrun job=backup level=full yes" | $bin/bconsole -c $tmp/bconsole-remote.conf + +cat <$tmp/bconcmds +@$out $tmp/log1.out +@sleep 10 +status dir +@sleep 10 +status dir +wait +messages +END_OF_DATA + +run_bconsole + +if test "$debug" -eq 1 ; then + echo -e "proxy\nrestore select all done yes" | $bin/bconsole -c $tmp/bconsole-remote.conf +else + echo -e "proxy\nrestore select all done yes" | $bin/bconsole -c $tmp/bconsole-remote.conf >/dev/null 2>&1 +fi + +sed -i 's/log1.out/log2.out/' $tmp/bconcmds + +run_bconsole +check_for_zombie_jobs storage=File1 + +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/restart-reschedule-test b/regress/tests/restart-reschedule-test new file mode 100755 index 0000000000..1504060cfd --- /dev/null +++ b/regress/tests/restart-reschedule-test @@ -0,0 +1,130 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test the RescheduleOnError function +# +TestName="restart-reschedule-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-migration-confs +scripts/prepare-disk-changer + +echo "${cwd}/build" >${cwd}/tmp/file-list +sed 's/migrate/copy/g' ${cwd}/bin/bacula-dir.conf > ${cwd}/tmp/1 +sed 's/Migrate/Copy/g' ${cwd}/tmp/1 > ${cwd}/bin/bacula-dir.conf + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "RescheduleOnError", "yes", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "RescheduleInterval", "10", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "RescheduleTimes", "3", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "LabelFormat", "Vol", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "LabelMedia", "Yes", "Device", "FileStorage")' + +change_jobname NightlySave $JobName +start_test + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +label volume=TestVolume001 pool=Scratch storage=DiskChanger slot=1 drive=0 +label volume=TestVolume002 pool=Scratch storage=DiskChanger slot=2 drive=0 +label volume=TestVolume003 pool=Scratch storage=DiskChanger slot=3 drive=0 +setdebug hangup=200 level=0 client +run job=$JobName yes +messages +@# Wait for the incomplete job +@sleep 5 +status dir +messages +@# Wait for the reschedule +wait +messages +@# +@# now do a restore +@# +@$out $tmp/log2.out +restore where=$tmp/bacula-restores select all done +yes +wait +messages +@$out $tmp/log3.out +@#setdebug hangup=200 level=0 storage +@# Test Reschedule with copy jobs +run job=copy-job jobid=1 yes +wait +messages +quit +END_OF_DATA + +run_bacula + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "RescheduleIncompleteJobs", "no", "Job")' + +cat <$tmp/bconcmds +@$out $tmp/log3.out +reload +@# Test stop/resume with incomplete +setbandwidth limit=1000 client +run job=$JobName jobid=1 level=full yes +@sleep 5 +stop jobid=5 +wait +setbandwidth limit=10000000 client +@sleep 2 +wait +messages +@# +@# now do a restore +@# +@$out $tmp/log5.out +restore where=$tmp/bacula-restores4 +3 +4 +m * +done +yes +wait +messages +@$out $tmp/log4.out +list jobs +status dir +quit +END_OF_DATA + +run_bconsole +check_for_zombie_jobs storage=File1 +stop_bacula + +check_two_logs +$rscripts/diff.pl -s $cwd/build -d $tmp/bacula-restores/$cwd/build +bstat=`expr $bstat + $?` +$rscripts/diff.pl -s $cwd/build -d $tmp/bacula-restores4/$cwd/build +bstat=`expr $bstat + $?` + +nb=`grep Incomplete $tmp/log4.out | wc -l` +if [ "$nb" -ne 1 ]; then + print_debug "ERROR: Expected 1 incomplete job in status dir output $tmp/log4.out, got $nb" + estat=1 +fi + +nb=`grep ' 5.* I ' $tmp/log4.out | wc -l` +if [ "$nb" -ne 1 ]; then + print_debug "ERROR: Expected 1 incomplete job in status dir output $tmp/log4.out, got $nb" + estat=1 +fi + + +end_test diff --git a/regress/tests/restart-sd-test b/regress/tests/restart-sd-test new file mode 100755 index 0000000000..0841d0498f --- /dev/null +++ b/regress/tests/restart-sd-test @@ -0,0 +1,76 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a backup of the build directory and check that it restarts correctly. +# +TestName="restart-sd-test" +JobName=RestartSD +. scripts/functions + +scripts/cleanup +scripts/copy-test-confs + +setup_shstore # simulate shared storage if enabled + +echo "${cwd}/build" >${cwd}/tmp/file-list + +change_jobname NightlySave $JobName +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log3.out +setbandwidth limit="1000 kb/s" client +@#setdebug level=20 dir +label storage=File volume=TestVolume001 slot=1 drive=0 +run job=$JobName yes +@sleep 10 +messages +quit +END_OF_DATA + +run_bacula + +pid=`cat $working/bacula-sd.*.pid` +if ps $pid | grep bacula-sd > /dev/null; then + kill -9 $pid +else + print_debug "ERROR: Unable to find bacula-sd pid, please review the script" + stop_bacula + end_test + exit 1 +fi + +sleep 5 + +$scripts/bacula-ctl-sd start + +cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log3.out +messages +cancel all yes +wait +messages +@$out ${cwd}/tmp/log1.out +setbandwidth limit="10000000 kb/s" client +run job=$JobName yes +wait +messages +@$out ${cwd}/tmp/log2.out +restore where=$tmp/bacula-restores storage=File select all done +yes +wait +messages +quit +END_OF_DATA + +run_bconsole +scripts/check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/restart2-base-job-test b/regress/tests/restart2-base-job-test new file mode 100755 index 0000000000..32eda89254 --- /dev/null +++ b/regress/tests/restart2-base-job-test @@ -0,0 +1,93 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a backup of the build directory but force it to have +# a comm error, and check that it restarts correctly. +# +# +TestName="restart-base-job-test" +JobName=RestartJobWithBase +. scripts/functions + +scripts/cleanup +scripts/copy-test-confs + +setup_shstore # simulate shared storage if enabled + +echo "${cwd}/build" >${cwd}/tmp/file-list + +change_jobname NightlySave $JobName + +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'Base', '$JobName', 'Job', '$JobName')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'Accurate', 'yes', 'Job', '$JobName')" +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=TestVolume001 slot=1 drive=0 +setdebug level=200 storage=File +setdebug level=0 trace=0 hangup=43 client +run job=$JobName level=Base yes +wait +messages +@exec "touch $cwd/build/po/fr.po" +setdebug level=200 trace=1 dir +setdebug level=0 trace=0 hangup=0 client +run job=$JobName yes +wait +messages +sql +select * from JobMedia; + +setdebug level=0 trace=0 dir +@$out ${cwd}/tmp/log3.out +list jobs +quit +END_OF_DATA + +run_bacula + +scripts/check_for_zombie_jobs storage=File + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +restore where=$tmp/bacula-restores storage=File select all done +yes +wait +messages +quit +END_OF_DATA + +run_bconsole +scripts/check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff + +#nb_B=`awk -F '|' '/RestartJob.+ B.+ B/ { gsub(/,/, "");print $7}' $tmp/log3.out` +#nb_F=`awk -F '|' '/RestartJob.+F/ { gsub(/,/, ""); print $7}' $tmp/log3.out` + +#if [ $nb_F -ne $nb_B ]; then +# print_debug "ERROR: Base and Full should have the same number of files" +# estat=1 +#fi + +#nb_B=`awk -F '|' '/RestartJob.+B.+B/ { gsub(/,/, ""); print $8}' $tmp/log3.out` +#nb_F=`awk -F '|' '/RestartJob.+F/ { gsub(/,/, ""); print $8}' $tmp/log3.out` + +#if [ $nb_F -gt 1000000 ]; then +# print_debug "ERROR: Base job should backup only one file" +# estat=2 +#fi + +end_test diff --git a/regress/tests/restart2-job-test b/regress/tests/restart2-job-test new file mode 100755 index 0000000000..4bdc904f6c --- /dev/null +++ b/regress/tests/restart2-job-test @@ -0,0 +1,129 @@ +#!/bin/sh +# +# Run a backup of the build directory but force it to have +# a comm error, and check that it restarts correctly. +# +TestName="restart2-job-test" +JobName=RestartJob +. scripts/functions + +scripts/cleanup +scripts/copy-test-confs +setup_shstore # simulate shared storage if enabled +echo "${cwd}/build" >${cwd}/tmp/file-list + +change_jobname NightlySave $JobName +$bperl -e 'add_attribute("$conf/bacula-fd.conf", "MaximumBandwidthPerJob", "100 kB/s", "FileDaemon")' + +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug level=0 trace=0 hangup=469 client +@#setdebug level=20 dir +label storage=File volume=TestVolume001 +run job=$JobName yes +@sleep 15 +status client +status storage=File +END_OF_DATA + +run_bacula 2>/dev/null + +# Note, the following stops and restarts the FD while it +# is running to simulate a crash, so it is normal if +# orphanned buffers are reported +#sleep 9 +$scripts/bacula-ctl-fd stop >/dev/null 2>&1 +sleep 5 +$scripts/bacula-ctl-fd start 1>/dev/null 2>&1 +sleep 5 + +cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log1.out +@sleep 15 +messages +quit +END_OF_DATA + +run_bconsole + +# Note, the following stops and restarts the FD while it +# is running to simulate a crash, so it is normal if +# orphanned buffers are reported +#sleep 9 +$scripts/bacula-ctl-fd stop >/dev/null 2>&1 +sleep 5 +$scripts/bacula-ctl-fd start >/dev/null 2>&1 +sleep 5 + +cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log1.out +@sleep 15 +messages +quit +END_OF_DATA + +run_bconsole + +# +# This is the last backup, so make it run in a reasonable time +# +$bperl -e 'add_attribute("$conf/bacula-fd.conf", "MaximumBandwidthPerJob", "500000 kB/s", "FileDaemon")' + +#sleep 9 +$scripts/bacula-ctl-fd stop >/dev/null 2>&1 +sleep 5 +$scripts/bacula-ctl-fd start +sleep 5 + +cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log1.out +wait +messages +sql +select * from JobMedia where JobId=1; + +quit +END_OF_DATA + + +run_bconsole + +scripts/check_for_zombie_jobs storage=File + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +@#setdebug level=50 storage=File +restore where=$tmp/bacula-restores storage=File select all done +yes +wait +messages +status dir +status client +status storage=File +messages +quit +END_OF_DATA + +run_bconsole +scripts/check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff + +grep "Sending Accurate" $tmp/log1.out >/dev/null +if [ $? != 0 ]; then + print_debug "ERROR: Second job should use Accurate" + estat=1 +fi + +end_test diff --git a/regress/tests/restore-multi-session-test b/regress/tests/restore-multi-session-test new file mode 100755 index 0000000000..d26d5602b6 --- /dev/null +++ b/regress/tests/restore-multi-session-test @@ -0,0 +1,138 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a two backup of the Bacula build directory + other files +# then restore one file and see if we have restore problems +# We need a large directory to store the volume +# MA 1150 +# +# The main goal is to have the following layout on the volume +# +# JobMedia <----------------- +# Job1 +# File1 +# File2 +# Job2 +# File1 +# Job1 +# File2 +# JobMedia <----------------- +# Job1 +# File2 +# +TestName="restore-multi-session-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumFileSize', '500MB', 'Device')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'MaximumVolumeBytes', '10GB', 'Pool')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'Label Format', 'Vol-', 'Pool', 'Default')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'FileSet', 'FewFiles', 'Job', 'BackupClient1')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'MaximumConcurrentJobs', 10, 'Job')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'MaximumConcurrentJobs', 10, 'Client')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'MaximumConcurrentJobs', 10, 'Storage')" +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumConcurrentJobs', 10, 'Device')" +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'CommCompression', 'no', 'Storage')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'CommCompression', 'no', 'Director')" +$bperl -e "add_attribute('$conf/bacula-fd.conf', 'CommCompression', 'no', 'FileDaemon')" +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumPartSize', '50MB', 'Device')" + +#$bperl -e "add_attribute('$conf/bacula-dir.conf', 'SpoolData', 'Yes', 'Job', 'BackupClient1')" +#$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumSpoolSize', '200MB', 'Device')" + +change_jobname BackupClient1 $JobName +start_test + +# If available, compress files on the FS, we will store only 0 +if [ "$FORCE_CLOUD" = "" ]; then + touch $tmp/Vol-0001 $tmp/Vol-0002 + + # might not be available + chattr +c $tmp/Vol-0001 2> /dev/null + chattr +c $tmp/Vol-0002 2> /dev/null +fi + +cat >> $conf/bacula-dir.conf < /dev/null + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +setbandwidth limit="82400 kb/s" client +run job=$JobName storage=File1 level=full yes +@sleep 2 +run job=$JobName storage=File1 level=full yes +wait +llist jobmedia jobid=1 +messages +END_OF_DATA + +run_bacula + +cat <$tmp/bconcmds +@$out $tmp/log2.out +@# +@# now do a restore +@# +setdebug level=500 trace=1 storage=File1 +restore where=$tmp/bacula-restores +3 +1 +cd "$tmp" +mark "1.dat" +done +@sleep 2 +@exec "sh -c 'cp $working/*.restore.1.bsr $working/restore1.bsr'" +yes +wait +messages +quit +END_OF_DATA + +run_bconsole + +check_for_zombie_jobs storage=File1 +stop_bacula + +check_two_logs + +cat $tmp/1.dat | md5sum > $tmp/1.dat.sum +cat $tmp/bacula-restores/$tmp/1.dat | md5sum > $tmp/1.dat.sum2 + +diff $tmp/1.dat.sum $tmp/1.dat.sum2 +rstat=$? + +end_test diff --git a/regress/tests/restore-stop-read-test b/regress/tests/restore-stop-read-test new file mode 100755 index 0000000000..d375637b6b --- /dev/null +++ b/regress/tests/restore-stop-read-test @@ -0,0 +1,145 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a simple backup of the Bacula build directory +# then restore one file and see if we read too much +# data +# +# With AWK, we get the 1st executable file from the volume we get the also a +# "position" of the file in the volume with NR (line number). +# +# With grep "FI=nn SessId=", we search in the trace file up to which FI we read the data. +# +# For example, if we find build/libtool NR=6, the trace file should have all +# FileIndex<6 (probably between 3 and 5) If we find a FI=6 in the trace file, +# it means that we read too much data + +TestName="restore-stop-read-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumFileSize', '10MB', 'Device')" +director=`$bperl -e "get_dirname()"` + +change_jobname BackupClient1 $JobName +start_test + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +run job=$JobName yes +wait +messages +END_OF_DATA + +run_bacula +stop_bacula + +$bin/bls -V TestVolume001 FileChgr1-Dev1 > $tmp/list + +f=`awk '/-rwxr-xr-x/ { print $11; exit }' $tmp/list` + +# The "line" in the bls is between 3-n, the fileindex (one by line) +# is between 1-n. So we search for FI=line, like FI=fileindex+3 +line=`awk '/-rwxr-xr-x/ { print NR; exit }' $tmp/list` + +if [ "x$f" = x ]; then + print_debug "ERROR: Unable to find a file in the backup list" + exit 1 +fi + +p=`dirname $f` +f=`basename $f` + +cat <$tmp/bconcmds +@$out $tmp/log2.out +@# +@# now do a restore +@# +setdebug level=500 trace=1 storage=File1 +restore where=$tmp/bacula-restores +5 +cd "$p" +mark "$f" +done +@sleep 2 +@exec "cp $working/$director.restore.1.bsr $working/restore1.bsr" +yes +wait +messages +setdebug level=0 trace=0 storage=File1 +quit +END_OF_DATA + +run_bacula + +print_debug "Looking for FI=$line in the SD working/*trace file" +# We have more or less one FI per line +grep "FI=$line SessId=" working/*trace > /dev/null +if [ $? -eq 0 ]; then + print_debug "ERROR: Should not find references to FileIndex $line in trace, should stop before" + estat=1 +fi + +# The "line" in the bls is between 3-n, the fileindex (one by line) +# is between 1-n. So we search for FI=line, like FI=fileindex+3 +f=`awk '/-rwxr-xr-x/ { if (count++ == 10) { print $11; exit} }' $tmp/list` +line=`awk '/-rwxr-xr-x/ { if(count++ == 10) {print NR; exit}}' $tmp/list` + +if [ "x$f" = x ]; then + print_debug "ERROR: Unable to find a file=$file in the backup list" + exit 1 +fi + +p=`dirname $f` +f=`basename $f` + +cat <$tmp/bconcmds +@$out $tmp/log2.out +@# +@# now do a restore +@# +setdebug level=500 trace=1 storage=File1 +restore where=$tmp/bacula-restores +5 +cd "$p" +mark "$f" +done +yes +wait +messages +setdebug level=0 trace=0 storage=File1 +quit +END_OF_DATA + +run_bconsole + +# We have more or less one FI per line +print_debug "Looking for file=$f FI=$line in the SD trace file" +grep "FI=$line SessId=" working/*trace > /dev/null +if [ $? -eq 0 ]; then + print_debug "ERROR: Should not find references to FileIndex $line in trace, should stop before" + estat=1 +fi + +check_for_zombie_jobs storage=File1 +stop_bacula + +check_two_logs +end_test diff --git a/regress/tests/restore-stop-read2-test b/regress/tests/restore-stop-read2-test new file mode 100755 index 0000000000..3b0b38260a --- /dev/null +++ b/regress/tests/restore-stop-read2-test @@ -0,0 +1,83 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run two simple backups of the Bacula build directory +# then restore the first one and see if we read too much +# data +# 1) run a full backup +# 2) run a 2nd full backup on the same volume +# 3) delete the 2nd job from the catalog +# 4) restore the first full backup +# 5) see in traces if we stopped at the end of the first backup +# +TestName="restore-stop-read2-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-confs + +# +# Zap out any schedule in default conf file so that +# it doesn't start during our test +# +outf="$tmp/sed_tmp" +echo "s% Schedule =%# Schedule =%g" >${outf} +cp $scripts/bacula-dir.conf $tmp/1 +sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf + +$bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumFileSize', '10MB', 'Device')" + +change_jobname BackupClient1 $JobName +start_test + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 +run job=$JobName level=full yes +wait +run job=$JobName level=full yes +wait +messages +END_OF_DATA + +run_bacula + +cat <$tmp/bconcmds +@$out $tmp/log2.out +@# +@# now do a restore +@# +setdebug level=500 trace=1 storage=File1 +delete jobid=2 yes +restore where=$tmp/bacula-restores select all done yes +wait +messages +setdebug level=0 trace=0 storage=File1 +quit +END_OF_DATA + +run_bconsole + +check_for_zombie_jobs storage=File1 +stop_bacula + +# looking for zog8-sd: dircmd.c:1243-3 VolAddr=99993804-100365372 +eaddr=`awk -F '-' '/VolAddr=/ { ret=$4 } END { print ret }' $working/*trace` + +# zog8-sd: match_bsr.c:674-3 match_voladdr: saddr=99993804 eaddr=100365372 recaddr=200358958 sfile=0 efile=0 recfile=0 +eaddrseen=`awk '/match_voladdr: saddr=/ { gsub(/recaddr=/, "", $6); ret=$6 } END { print ret}' $working/*trace` + +eaddrmax=$(($eaddr + $eaddr/2)) + +if [ "$eaddrseen" -gt "$eaddrmax" ]; then + print_debug "ERROR: The restore process read too much data should stop at eaddr=$eaddr, read eaddrseen=$eaddrseen" + estat=1 +fi + +check_two_logs +end_test diff --git a/regress/tests/scratch-pool-test b/regress/tests/scratch-pool-test index 17274e1648..4e4d71b6d2 100755 --- a/regress/tests/scratch-pool-test +++ b/regress/tests/scratch-pool-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a simple backup of the Bacula build directory # to two tapes where the maximum tape file size is set to 1M diff --git a/regress/tests/scratchpool-pool-test b/regress/tests/scratchpool-pool-test index 401b8f7772..da9f76a401 100755 --- a/regress/tests/scratchpool-pool-test +++ b/regress/tests/scratchpool-pool-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a simple backup of the Bacula build directory # to two tapes where the maximum tape file size is set to 1M diff --git a/regress/tests/sd-sd-test b/regress/tests/sd-sd-test new file mode 100755 index 0000000000..079ef9b4c1 --- /dev/null +++ b/regress/tests/sd-sd-test @@ -0,0 +1,136 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test SD to SD copy/migration +# +TestName="sd-sd-test" +JobName=MigrationJobSave + +# the procedure to test volumes is not working with 2 SD +REGRESS_CHECK_CORRUPTION=no +. scripts/functions + + +scripts/cleanup +scripts/copy-migration-confs +scripts/prepare-disk-changer +echo "${cwd}/build" >${cwd}/tmp/file-list + +sed -e "s:$cwd/working:$cwd/working2: " \ + -e "s:$cwd/tmp:$cwd/tmp2:" $conf/bacula-sd.conf > $conf/bacula-sd2.conf + +$bperl -e "add_attribute('$conf/bacula-sd2.conf', 'Name', 'sd2', 'Storage')" +$bperl -e "add_attribute('$conf/bacula-sd2.conf', 'SDPort', $BASEPORT + 10, 'Storage')" +$bperl -e "add_attribute('$conf/bacula-sd2.conf', 'Media Type', 'DiskChangerMedia2', 'Device', 'Drive-0')" +$bperl -e "add_attribute('$conf/bacula-sd2.conf', 'Media Type', 'File2', 'Device', 'FileStorage')" +if [ x$FORCE_DEDUP = xyes ]; then + DEDUP_FS_OPTION=${DEDUP_FS_OPTION:-bothsides} + $bperl -e 'add_attribute("$conf/bacula-sd2.conf", "Plugin Directory", "$plugins", "Storage")' + $bperl -e 'add_attribute("$conf/bacula-sd2.conf", "Device Type", "Dedup", "Device")' + $bperl -e 'add_attribute("$conf/bacula-sd2.conf", "DedupDirectory", "${working}2", "Storage")' + $bperl -e 'add_attribute("$conf/bacula-sd2.conf", "MaximumContainerSize", "50MB", "Storage")' +fi + + +rm -rf $cwd/working2 $cwd/tmp2 +mkdir -p $cwd/working2 $cwd/tmp2 +cp -r $tmp/disk-changer $cwd/tmp2 + +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'SDPort', $BASEPORT + 10, 'Storage', 'File')" +$bperl -e "add_attribute('$conf/bacula-dir.conf', 'Media Type', 'File2', 'Storage', 'File')" + +# Here, File is pointing to SD2 and DiskChanger is pointing to SD1 + +$bin/bacula-sd -c $conf/bacula-sd2.conf +sleep 1 +pid=`cat $cwd/working2/bacula-sd.*.pid` +trap "kill $pid" INT + + +change_jobname NightlySave $JobName +start_test + +# +# Note, we first backup into Pool Default, +# then Migrate into Pool Full. +# Pool Default uses Storage=File (SD2) +# Pool Full uses Storage=DiskChanger (SD1) + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=FileVolume001 Pool=Default +label storage=DiskChanger volume=ChangerVolume001 slot=1 Pool=Full drive=0 +label storage=DiskChanger volume=ChangerVolume002 slot=2 Pool=Full drive=0 +@# run two jobs (both will be migrated) +run job=$JobName yes +run job=$JobName yes +wait +list jobs +list volumes +@#setdebug level=100 dir +@# should migrate two jobs +@#setdebug level=51 storage=DiskChanger +messages +st storage=File +st storage=DiskChanger +@# setdebug level=200 dir +@# setdebug level=200 storage=File +@# setdebug level=200 storage=DiskChanger +setdebug level=700 storage=File trace=1 flags=dedup,asx,dde +setdebug level=700 storage=DiskChanger trace=1 flags=dedup,asx,dde +run job=migrate-job yes +@sleep 10 +messages +wait +messages +@# purge volume=FileVolume001 +list jobs +list volumes +list joblog jobid=6 +wait +@# +@# Now do another backup, but level Incremental +@# +run job=$JobName level=Incremental yes +wait +messages +@# +@# This final job that runs should be Incremental and +@# not upgraded to full. +list jobs +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +list volumes +restore where=${cwd}/tmp/bacula-restores select storage=DiskChanger +unmark * +mark * +done +yes +list volumes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff + +trap - INT + +PID=`cat $cwd/working2/bacula-sd.*.pid` +if [ "$PID" != "" ]; then + kill $PID >/dev/null 2>/dev/null +fi + +end_test diff --git a/regress/tests/source-addr-test b/regress/tests/source-addr-test index 48c746ff9a..757646e251 100755 --- a/regress/tests/source-addr-test +++ b/regress/tests/source-addr-test @@ -1,9 +1,7 @@ #!/bin/sh # -# Copyright (C) 2000-2015 Kern Sibbald +# Copyright (C) 2000-2017 Kern Sibbald # License: BSD 2-Clause; see file LICENSE-FOSS -# - # # Run a simple backup of the Bacula build directory # then restore it. @@ -69,7 +67,7 @@ messages @$out ${cwd}/tmp/log1.out @#label volume=TestVolume001 storage=File1 pool=File slot=1 drive=0 @#setdebug level=100 storage=File1 -setbandwidth client limit=1000 +setbandwidth client limit="1000 kb/s" run job=$JobName yes @sleep 3 messages @@ -102,8 +100,8 @@ if [ "$debug" = 1 ] ; then fi cat <${cwd}/tmp/bconcmds -@output /dev/null -setbandwidth client limit=1000000 +@$out $tmp/log1.out +setbandwidth client limit="1000000 kb/s" wait messages @# diff --git a/regress/tests/three-pool-virtual-test b/regress/tests/three-pool-virtual-test new file mode 100755 index 0000000000..4eeae3af0e --- /dev/null +++ b/regress/tests/three-pool-virtual-test @@ -0,0 +1,89 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a simple backup of the Bacula build directory. Create three +# tapes, each in a different pool, then run two jobs both of which +# want the tape that is not loaded. Note, they both have +# prefers non-mounted tapes. This should expose bug #801 +# +# This script uses the virtual disk autochanger and two drives +# +TestName="three-pool-virtual-test" +JobName="threepooltest" +. scripts/functions + +scripts/cleanup +scripts/copy-2disk-tape-confs +CLIENT=2drive2disk + +echo "${cwd}/build" >${cwd}/tmp/file-list +change_jobname NightlySave $JobName +start_test + +# Turn off Prefer Mounted Volumes so we use 2 drives +# Turn off spooling +outf="${cwd}/tmp/sed_tmp" +echo "s%# Prefer Mounted Volumes% Prefer Mounted Volumes%g" >${outf} +echo "s% SpoolData = yes%# SpoolData = yes%g" >>${outf} +cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1 +# Comment the next line out to write everything to one drive +# otherwise, it writes the two jobs to different drives +sed -f ${outf} ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@out /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug level=150 storage=tape +@#setdebug level=100 client=$CLIENT +label storage=tape volume=TestVolume001 slot=1 Pool=Default drive=0 +label storage=tape volume=TestVolume002 slot=2 Pool=Full drive=0 +label storage=tape volume=TestVolume003 slot=3 Pool=Inc drive=1 +status storage=tape +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +run job=$JobName level=Full Pool=Default yes +@#setdebug level=20 storage=tape +wait +sql +select jobid,firstindex,lastindex,startblock,endblock from JobMedia; + +list volumes +list jobs +status storage=tape +messages +quit +END_OF_DATA + +# exit + +run_bacula +cat <${cwd}/tmp/bconcmds +@out /dev/null +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +setdebug level=150 storage=tape +restore where=${cwd}/tmp/bacula-restores select all storage=tape done +yes +wait +messages +quit +END_OF_DATA + +run_bconsole + +check_for_zombie_jobs storage=tape +stop_bacula + +check_two_logs +check_restore_diff + +end_test diff --git a/regress/tests/truncate-bug-tape b/regress/tests/truncate-bug-tape index d5df7a720b..f144460610 100755 --- a/regress/tests/truncate-bug-tape +++ b/regress/tests/truncate-bug-tape @@ -28,7 +28,7 @@ messages label storage=tape volume=TestVolume001 slot=0 pool=Default @# do a bunch of saves so we have 12 files on the tape @#setdebug level=100 dir -setdebug level=100 storage=tape +setdebug level=50 storage=tape run job=$JobName yes run level=Full job=$JobName yes run level=Full job=$JobName yes diff --git a/regress/tests/truncate-test b/regress/tests/truncate-test new file mode 100755 index 0000000000..a86e38d867 --- /dev/null +++ b/regress/tests/truncate-test @@ -0,0 +1,221 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test truncate command (replaces old purge action=truncate ...) +# +# +TestName="truncate-test" +JobName=FIFOTest +. scripts/functions + +cwd=`pwd` +scripts/cleanup +scripts/copy-test-confs + +echo $src > $tmp/file-list + +sed 's/Pool Type = Backup/Pool Type = Backup; ActionOnPurge = Truncate/' $conf/bacula-dir.conf > $tmp/1 +cp $tmp/1 $conf/bacula-dir.conf + +start_test + +cat >tmp/bconcmds < $tmp/bconcmds +run_bconsole + +cd $tmp + +if test "$debug" -eq 1 ; then + echo "Volume TestVolume001 should be truncated ..." + ls -l TestVolume* +fi + +for i in 2 3 4 5 ; do + size=`du -b TestVolume00$i|cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 3: Volume TestVolume00$i is too small $size" + ls -l TestVolume00$i + estat=3 + fi +done +for i in 1 ; do + size=`du -b TestVolume00${i} | cut -f1` + if test $size -gt 5000 ; then + print_debug "ERROR 4: Volume TestVolume00$i is not truncated (too big) $size" + ls -l TestVolume00$i + estat=4 + fi +done + +cd $cwd + +echo "truncate volume storage=File" > $tmp/bconcmds +run_bconsole + +cd $tmp +if test "$debug" -eq 1 ; then + echo "Volumes 001, 003, and 004 should be truncated ..." + ls -l TestVolume* +fi + +for i in 2 5 ; do + size=`du -b TestVolume00$i | cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 5: Volume TestVolume00$i is too small $size" + ls -l TestVolume00$i + estat=5 + fi +done +for i in 1 3 4 ; do + size=`du -b TestVolume00$i | cut -f1` + if test $size -gt 5000 ; then + print_debug "ERROR 6: Volume TestVolume00$i is not truncated (too big) $size" + ls -l TestVolume00$i + estat=6 + fi +done + +cd $cwd + +cat < $tmp/bconcmds +@######################################################### +@# Display catalog settings for Pool and Media +@######################################################### +@$out $tmp/log4.out +setdebug level=0 director +sql +select VolumeName, ActionOnPurge FROM Media; +select Name, ActionOnPurge FROM Pool; + +wait +quit +END_OF_DATA + +run_bconsole + +stop_bacula + +touch $tmp/log2.out +check_two_logs + +print_debug "Test if Pool record is ok" +r=`awk '/Default/ { print $4 }' $tmp/log4.out` +if [ "$r" != 1 ]; then + print_debug "ERROR 5: ActionOnPurge on Pool record should be 1 ($r)" + estat=5 +fi + +print_debug "Test TestVolume001 if Media record is ok" +r=`awk '/TestVolume001/ { print $4 }' $tmp/log4.out` +if [ "$r" != 1 ]; then + print_debug "ERROR 6: ActionOnPurge on Media record should be 1" + estat=6 +fi + +print_debug "Test TestVolume002 if Media record is ok" +r=`awk '/TestVolume002/ { print $4 }' $tmp/log4.out` +if [ "$r" != 0 ]; then + print_debug "ERROR 7: ActionOnPurge on Media record should be 0" + estat=7 +fi + +end_test diff --git a/regress/tests/truncate2-test b/regress/tests/truncate2-test new file mode 100755 index 0000000000..33ba0bd5fc --- /dev/null +++ b/regress/tests/truncate2-test @@ -0,0 +1,229 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test truncate command (replaces old purge action=truncate ...) +# +# We run several jobs on different volumes, than during the last job +# we run a truncate command. +# +# +TestName="truncate2-test" +JobName=NightlySave +. scripts/functions + +cwd=`pwd` +scripts/cleanup +scripts/copy-2disk-virtual-confs + +echo $src > $tmp/file-list + +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "ActionOnPurge", "Truncate", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "SpoolData", "No", "Job")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "Maximum Volume Size", "500M", "Device")' +start_test + +cat >tmp/bconcmds < $tmp/bconcmds +@$out $tmp/log.err +truncate volume=TestVolume001 storage=vDrive-1 allpools +@$out $tmp/log3.out +truncate volume=TestVolume001 storage=vDrive-2 allpools +EOF + +run_bconsole + +if test "$debug" -eq 1 ; then + echo "Volume TestVolume001 should be truncated ..." + cd ${tmp} + ls -l TestVolume* + cd ${cwd} +fi + +for i in 2 3 4 5 ; do + size=`du -b TestVolume00$i|cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 3: Volume TestVolum00$i is too small $size" + ls -l TestVolume00$i + estat=3 + fi +done +for i in 1 ; do + size=`du -b TestVolume00$i|cut -f1` + if test $size -gt 5000 ; then + print_debug "ERROR 4: Volume TestVolum00$i is not truncated (too big) $size" + ls -l TestVolume00$i + estat=4 + fi +done + +echo "truncate volume storage=vDrive-2 allpools" > $tmp/bconcmds +run_bconsole + +if test "$debug" -eq 1 ; then + echo "Volumes 001, 003, and 004 should be truncated ..." + cd ${tmp} + ls -l TestVolume* + cd ${cwd} +fi + +for i in 2 5 ; do + size=`du -b TestVolume00$i|cut -f1` + if test $size -lt 5000 ; then + print_debug "ERROR 5: Volume TestVolum00$i is too small $size" + ls -l TestVolume00$i + estat=5 + fi +done +for i in 1 3 4 ; do + size=`du -b TestVolume00$i|cut -f1` + if test $size -gt 5000 ; then + print_debug "ERROR 6: Volume TestVolum00$i is not truncated (too big) $size" + ls -l TestVolume00$i + estat=6 + fi +done + + +cat < $tmp/bconcmds +@$out $tmp/log1.out +setdebug level=0 dir +setbandwidth limit="1000000 kb/s" client +wait +messages +@$out $tmp/log2.out +restore jobid=5 all done yes where=$tmp/bacula-restores +wait +messages +@######################################################### +@# Display catalog settings for Pool and Media +@######################################################### +@$out $tmp/log4.out +setdebug level=0 director +sql +select VolumeName, ActionOnPurge FROM Media; +select Name, ActionOnPurge FROM Pool; + +wait +quit +END_OF_DATA + +run_bconsole + +check_for_zombie_jobs storage=File + +stop_bacula + +check_two_logs + +check_restore_diff + +print_debug "Test if Pool record is ok" +r=`awk '/Default/ { print $4 }' $tmp/log4.out` +if [ "$r" != 1 ]; then + print_debug "ActionOnPurge on Pool record should be 1 ($r)" + estat=2 +fi + +print_debug "Test TestVolume001 if Media record is ok" +r=`awk '/TestVolume001/ { print $4 }' $tmp/log4.out` +if [ "$r" != 1 ]; then + print_debug "ActionOnPurge on Media record should be 1" + estat=2 +fi + +print_debug "Test TestVolume002 if Media record is ok" +r=`awk '/TestVolume002/ { print $4 }' $tmp/log4.out` +if [ "$r" != 0 ]; then + print_debug "ActionOnPurge on Media record should be 0" + estat=2 +fi + +end_test diff --git a/regress/tests/two-jobs-test b/regress/tests/two-jobs-test index c28fb06229..2b27100346 100755 --- a/regress/tests/two-jobs-test +++ b/regress/tests/two-jobs-test @@ -51,7 +51,7 @@ END_OF_DATA run_bacula check_for_zombie_jobs storage=File -echo "Backup 1 done" +#echo "Backup 1 done" #$bin/bls -d 4 -v -c $conf/bacula-sd.conf -V 'TestVolume001' FileStorage touch ${cwd}/build/src/dird/dird_conf.c # diff --git a/regress/tests/two-pool-changer b/regress/tests/two-pool-changer index ea86818522..7d8afad825 100755 --- a/regress/tests/two-pool-changer +++ b/regress/tests/two-pool-changer @@ -21,9 +21,7 @@ require_autochanger scripts/cleanup scripts/copy-2tape-confs -echo "Prepare two tapes" scripts/prepare-two-tapes -echo "Done prepare two tapes" # Make a relatively large backup set 5 x source code directory echo "${cwd}/build" >${cwd}/tmp/file-list diff --git a/regress/tests/virtual-jobid-test b/regress/tests/virtual-jobid-test new file mode 100755 index 0000000000..ac26e95edd --- /dev/null +++ b/regress/tests/virtual-jobid-test @@ -0,0 +1,215 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Run a simple backup of the Bacula build directory then do a +# Virtual Full backup to another device. +# +# This script uses the disk autochanger +# +TestName="virtual-jobid-test" +JobName=Vbackup +. scripts/functions + + +scripts/cleanup +scripts/copy-migration-confs +echo "${cwd}/build" >${cwd}/tmp/file-list + +rm -f $cwd/build/inc1 $cwd/build/inc2 $cwd/build/diff1 + +change_jobname NightlySave $JobName +start_test + +# +# Note, we first backup into Pool Default, +# then Migrate into Pool Full. +# Pool Default uses Storage=File +# Pool Full uses Storage=DiskChanger + +# +-------+---------+-------+----------+----------+-----------+ +# | JobId | Name | Level | JobFiles | JobBytes | JobStatus | +# +-------+---------+-------+----------+----------+-----------+ +# | 1 | Vbackup | F | 1754 | 50118554 | T | +# | 2 | Vbackup | I | 1 | 0 | T | +# | 3 | Vbackup | D | 2 | 0 | T | +# | 4 | Vbackup | I | 1 | 0 | T | +# | 5 | Save | F | 1754 | 50118554 | T | +# +-------+---------+-------+----------+----------+-----------+ + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug level=100 storage=File +label storage=File volume=FileVolume001 Pool=Default +label storage=DiskChanger volume=ChangerVolume001 slot=1 Pool=Full drive=0 +label storage=DiskChanger volume=ChangerVolume002 slot=2 Pool=Full drive=0 +@# run several jobs +@exec "sh -c 'date > ${cwd}/build/date'" +run job=$JobName level=Full yes +wait +messages +list jobs +@exec "sh -c 'touch ${cwd}/build/inc1'" +run job=$JobName level=Incremental yes +wait +messages +list jobs +@exec "sh -c 'touch ${cwd}/build/diff1'" +run job=$JobName level=Differential yes +wait +messages +list jobs +@exec "sh -c 'touch ${cwd}/build/inc2'" +run job=$JobName level=Incremental yes +wait +messages +list jobs +run job=Save level=Full yes +wait +messages +list jobs +@# should Consolidate Full, Incremental +@$out $tmp/log5.out +setdebug level=100 dir trace=1 +run job=$JobName jobid=4 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log6.out +run job=$JobName jobid=3 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log7.out +run job=$JobName jobid=2 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log8.out +run job=$JobName jobid=1,2 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log9.out +run job=$JobName jobid=1,3 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log10.out +run job=$JobName jobid=1,3,4 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log11.out +run job=$JobName jobid=1-5 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log12.out +run job=$JobName jobid=2-5 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log13.out +run job=$JobName jobid=2,4 level=VirtualFull yes +wait +messages +list jobs +@$out $tmp/log14.out +run job=$JobName alljobid=1-5 level=VirtualFull yes +wait +messages +list jobs +@# +@# now do a restore of the consolidated Full +@# +@$out $tmp/log2.out +restore jobid=8 where=${cwd}/tmp/restore1 +m * +done +yes +wait +messages +restore jobid=7 where=${cwd}/tmp/restore2 +m * +done +yes +wait +messages +restore jobid=6 where=${cwd}/tmp/bacula-restores +m * +done +yes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +# +# We only used one log so copy it to the second log +# so that any restore errors will be picked up +# +check_two_logs +check_restore_diff + +grep 'JobIds=1,2,3,4$' $tmp/log11.out > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: jobid list is not right in $tmp/log11.out" + estat=1 +fi + +grep 'JobIds=2,3,4$' $tmp/log12.out > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: jobid list is not right in $tmp/log12.out" + estat=1 +fi + +grep 'using Differential level' $tmp/log12.out > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: Final level should be differential in $tmp/log12.out" + estat=1 +fi + +grep 'JobIds=2,4$' $tmp/log13.out > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: jobid list is not right in $tmp/log13.out" + estat=1 +fi + +grep 'using Incremental level' $tmp/log13.out > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: Final level should be incremental in $tmp/log13.out" + estat=1 +fi + +grep 'JobIds=1,2,3,4,5$' $tmp/log14.out > /dev/null +if [ $? -ne 0 ]; then + print_debug "ERROR: jobid list is not right in $tmp/log14.out" + estat=1 +fi + + +if [ ! -f $tmp/restore1/$cwd/build/inc1 -o -f $tmp/restore1/$cwd/build/diff1 -o -f $tmp/restore1/$cwd/build/inc2 ]; then + print_debug "ERROR: should find only inc1 in restore1" + estat=1 +fi + +if [ ! -f $tmp/restore2/$cwd/build/inc1 -o ! -f $tmp/restore2/$cwd/build/diff1 -o -f $tmp/restore2/$cwd/build/inc2 ]; then + print_debug "ERROR: should find only inc1 and diff1 in restore2" + estat=2 +fi + +if [ ! -f $tmp/bacula-restores/$cwd/build/inc1 -o ! -f $tmp/bacula-restores/$cwd/build/diff1 -o ! -f $tmp/bacula-restores/$cwd/build/inc2 ]; then + print_debug "ERROR: should find inc1, diff1 and inc2 in bacula-restores" + estat=2 +fi + +end_test diff --git a/regress/tests/virtualfull-bug-7154 b/regress/tests/virtualfull-bug-7154 new file mode 100755 index 0000000000..12cb37c292 --- /dev/null +++ b/regress/tests/virtualfull-bug-7154 @@ -0,0 +1,85 @@ +#!/bin/sh +# +# Copyright (C) 2000-2017 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# This should expose bug #7154 +# +# This script uses the virtual disk autochanger and two drives +# +TestName="virtualfull-bug-7154" +JobName="backup" +. scripts/functions + +scripts/cleanup +scripts/copy-2disk-tape-confs +CLIENT=2drive2disk + +echo "${cwd}/build" >${cwd}/tmp/file-list +change_jobname NightlySave $JobName +start_test + +# Turn on automatic label +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "LabelFormat", "vol", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "LabelMedia", "yes", "Device")' + + +# Enable nextpool for virtualfull and accurate fag +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Accurate", "yes", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "BackupsToKeep", "5", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "DeleteConsolidatedJobs", "yes", "Job")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "Storage", "tape", "Pool")' +$bperl -e 'add_attribute("$conf/bacula-dir.conf", "NextPool", "Default", "Pool", "Default")' + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug level=100 client=$CLIENT +run job=$JobName level=Full Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@exec "touch $cwd/build/po/fr.po" +run job=$JobName level=Incremental Pool=Default yes +wait +@#setdebug level=150 storage=tape trace=1 +run job=$JobName level=VirtualFull yes +wait +messages +list jobs +messages +quit +END_OF_DATA + +# exit + +run_bacula + +check_for_zombie_jobs storage=tape +stop_bacula + +touch $tmp/log2.out +check_two_logs + +end_test diff --git a/regress/tests/vtape-round-robin-changer b/regress/tests/vtape-round-robin-changer new file mode 100755 index 0000000000..d93c98e594 --- /dev/null +++ b/regress/tests/vtape-round-robin-changer @@ -0,0 +1,97 @@ +#!/bin/sh +# +# Run backups with dummy tape driver +# This test setups an Autochanger with 80 slots +# and 5 drives (3 LTO3 and 2 LTO1) +# +# TAPE_DRIVE="$cwd/working/ach/drive0" +# TAPE_DRIVE1="$cwd/working/ach/drive0" +# AUTOCHANGER="$cwd/working/ach/conf" +# USE_VTAPE=yes +# AUTOCHANGER_SCRIPT=disk-changer +# + +TestName="vtape-round-robin-changer" +JobName=backup +. scripts/functions + +require_vtape + +scripts/cleanup +scripts/copy-tape-confs +cp $rscripts/bacula-dir-vtape.conf $conf/bacula-dir.conf +cp $rscripts/bacula-sd-vtape.conf $conf/bacula-sd.conf +scripts/prepare-fake-autochanger + +echo "${cwd}/build" >${cwd}/tmp/file-list + +start_test + +clientname=`awk '/Name = .*-fd/ { if (!ok) { print $3 ; ok=1 } }' bin/bacula-dir.conf` + +# Catalog record for cleaning tape "CLN01" successfully created. +# CLN01 | Cleaning + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@$out /dev/null +messages +@$out ${cwd}/tmp/log6.out +@#setdebug level=200 storage=LTO1 +label barcodes pool=Scratch slots=1-40 storage=LTO3 drive=0 +yes +messages +list volumes +END_OF_DATA + +run_bacula + +cat <${cwd}/tmp/bconcmds +@$out $tmp/log1.out +run storage=LTO3 job=NightlySave spooldata=no pool=Inc yes +wait +messages +@sleep 3 +@$out $tmp/log3.out +@#setdebug level=150 Storage=LTO3 +run storage=LTO3 job=NightlySave spooldata=no level=full pool=Inc yes +wait +setdebug level=0 Storage=LTO3 +messages +quit +END_OF_DATA + +run_bconsole + +cat <${cwd}/tmp/bconcmds +@$out $tmp/log2.out +@# +@# now do a restore +@# +restore client=$clientname fileset="Full Set" pool=Inc where=${cwd}/tmp/bacula-restores select all done +yes +wait +messages +wait +messages +END_OF_DATA + +run_bconsole + +stop_bacula + +check_two_logs +check_restore_diff + +# Get the first slot used +s=`awk '/Issuing autochanger/ { print $13;exit }' $tmp/log3.out` + +# count how many times we unload this volume +# Note the second job output is in log3.out +nb=`grep "unload .* Slot $s" $tmp/log3.out | wc -l` +if [ $nb -gt 1 ]; then + estat=1 + print_debug "ERROR: Found $nb 'unload slot $s' instead of 1 for the second job in $tmp/log3.out" +fi + +end_test -- 2.39.5