3 # Test for a bug that causes massively parallel virtual full jobs to
4 # deadlock. First create a lot of backups, then run
5 # virtual fulls on them.
7 TestName="virtualfull-extreme-concurrency-bug-test"
12 scripts/copy-test-confs
13 cp scripts/virtualfull-extreme-bacula-dir.conf bin/bacula-dir.conf
14 cp scripts/migrate-bacula-sd.conf bin/bacula-sd.conf
16 # we have 20 distinct jobs. Each of those is run once at full level,
17 # then four times incremental, each time adding data so we end up with
19 # We should have jobs where several virtual fulls compete for volumes.
22 # Backup Bacula stored+objects directory
26 echo "${tmpsrc}" >${tmp}/file-list
27 for i in `ls ${src}/src/stored/*` ; do
30 # Get path to temp source files to restore
34 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
37 @$out ${cwd}/tmp/log1.out
38 @# No prelabeled volumes needed since they are automatically created
39 @#label storage=File slot=0 drive=0 Pool=Hot1 volume=Vol1
43 # Start Bacula and do label
46 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
49 @$out ${cwd}/tmp/log0.out
50 run job=HotV01 level=Full yes
51 run job=HotV02 level=Full yes
52 run job=HotV03 level=Full yes
53 run job=HotV04 level=Full yes
54 run job=HotV05 level=Full yes
55 run job=HotV06 level=Full yes
56 run job=HotV07 level=Full yes
57 run job=HotV08 level=Full yes
58 run job=HotV09 level=Full yes
59 run job=HotV10 level=Full yes
60 run job=HotV11 level=Full yes
61 run job=HotV12 level=Full yes
62 run job=HotV13 level=Full yes
63 run job=HotV14 level=Full yes
64 run job=HotV15 level=Full yes
65 run job=HotV16 level=Full yes
66 run job=HotV17 level=Full yes
67 run job=HotV18 level=Full yes
68 run job=HotV19 level=Full yes
69 run job=HotV20 level=Full yes
77 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
82 # add more files to backup source, so following incrementals
83 # have something to back up
84 for a in wx-console filed lib tools ; do
85 #echo ">>> adding ${a} to backup"
86 for i in `ls ${src}/src/${a}/*` ; do
87 cp -p "$i" ${tmpsrc} >/dev/null 2>&1
89 #echo "now $(ls ${tmpsrc}|wc -l) files to back up"
91 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
92 @$out ${cwd}/tmp/log1.out a
94 run job=HotV01 level=Incremental yes
95 run job=HotV02 level=Incremental yes
96 run job=HotV03 level=Incremental yes
97 run job=HotV04 level=Incremental yes
98 run job=HotV05 level=Incremental yes
99 run job=HotV06 level=Incremental yes
100 run job=HotV07 level=Incremental yes
101 run job=HotV08 level=Incremental yes
102 run job=HotV09 level=Incremental yes
103 run job=HotV10 level=Incremental yes
104 run job=HotV11 level=Incremental yes
105 run job=HotV12 level=Incremental yes
106 run job=HotV13 level=Incremental yes
107 run job=HotV14 level=Incremental yes
108 run job=HotV15 level=Incremental yes
109 run job=HotV16 level=Incremental yes
110 run job=HotV17 level=Incremental yes
111 run job=HotV18 level=Incremental yes
112 run job=HotV19 level=Incremental yes
113 run job=HotV20 level=Incremental yes
122 # now do the final virtual full levels.
123 #echo ">>>> Now the 1st VFull"
125 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
128 @$out ${cwd}/tmp/log1.out a
129 @#setdebug level=100 storage
130 run job=HotV01 level=VirtualFull yes
131 run job=HotV02 level=VirtualFull yes
132 run job=HotV03 level=VirtualFull yes
133 run job=HotV04 level=VirtualFull yes
134 run job=HotV05 level=VirtualFull yes
135 run job=HotV06 level=VirtualFull yes
136 run job=HotV07 level=VirtualFull yes
137 run job=HotV08 level=VirtualFull yes
138 run job=HotV09 level=VirtualFull yes
139 run job=HotV10 level=VirtualFull yes
140 run job=HotV11 level=VirtualFull yes
141 run job=HotV12 level=VirtualFull yes
142 run job=HotV13 level=VirtualFull yes
143 run job=HotV14 level=VirtualFull yes
144 run job=HotV15 level=VirtualFull yes
145 run job=HotV16 level=VirtualFull yes
146 run job=HotV17 level=VirtualFull yes
147 run job=HotV18 level=VirtualFull yes
148 run job=HotV19 level=VirtualFull yes
149 run job=HotV20 level=VirtualFull yes
152 select pool.name as poolname,volumename,volstatus,volbytes,voljobs,lastwritten from Pool,Media where Pool.PoolId=Media.MediaId order by poolname,volumename;
158 # Run Virtual Full Jobs
161 # add more files to backup source, so following incrementals
162 # have something to back up
163 for a in `seq 1 12` ; do
164 #echo ">>> adding files to backup: ${a}"
165 for b in `seq 1 150` ; do
166 date > "${tmpsrc}/run${a}-${b}.data"
168 #echo "now $(ls ${tmpsrc}|wc -l) files to back up"
170 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
171 @$out ${cwd}/tmp/log1.out a
173 run job=HotV01 level=Incremental yes
174 run job=HotV02 level=Incremental yes
175 run job=HotV03 level=Incremental yes
176 run job=HotV04 level=Incremental yes
177 run job=HotV05 level=Incremental yes
178 run job=HotV06 level=Incremental yes
179 run job=HotV07 level=Incremental yes
180 run job=HotV08 level=Incremental yes
181 run job=HotV09 level=Incremental yes
182 run job=HotV10 level=Incremental yes
183 run job=HotV11 level=Incremental yes
184 run job=HotV12 level=Incremental yes
185 run job=HotV13 level=Incremental yes
186 run job=HotV14 level=Incremental yes
187 run job=HotV15 level=Incremental yes
188 run job=HotV16 level=Incremental yes
189 run job=HotV17 level=Incremental yes
190 run job=HotV18 level=Incremental yes
191 run job=HotV19 level=Incremental yes
192 run job=HotV20 level=Incremental yes
202 ls ${tmpsrc}/* >${tmp}/restore-list
204 # now do the final virtual full levels.
205 #echo ">>>> Now the 2nd VFull"
207 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
210 @$out ${cwd}/tmp/log1.out a
211 @#setdebug level=100 storage
212 run job=HotV01 level=VirtualFull yes
213 run job=HotV02 level=VirtualFull yes
214 run job=HotV03 level=VirtualFull yes
215 run job=HotV04 level=VirtualFull yes
216 run job=HotV05 level=VirtualFull yes
217 run job=HotV06 level=VirtualFull yes
218 run job=HotV07 level=VirtualFull yes
219 run job=HotV08 level=VirtualFull yes
220 run job=HotV09 level=VirtualFull yes
221 run job=HotV10 level=VirtualFull yes
222 run job=HotV11 level=VirtualFull yes
223 run job=HotV12 level=VirtualFull yes
224 run job=HotV13 level=VirtualFull yes
225 run job=HotV14 level=VirtualFull yes
226 run job=HotV15 level=VirtualFull yes
227 run job=HotV16 level=VirtualFull yes
228 run job=HotV17 level=VirtualFull yes
229 run job=HotV18 level=VirtualFull yes
230 run job=HotV19 level=VirtualFull yes
231 run job=HotV20 level=VirtualFull yes
234 select pool.name as poolname,volumename,volstatus,volbytes,voljobs,lastwritten from Pool,Media where Pool.PoolId=Media.MediaId order by poolname,volumename;
240 # Run Virtual Full Jobs
242 cat <<END_OF_DATA >${cwd}/tmp/bconcmds
245 @$out ${cwd}/tmp/log2.out
246 restore client=HotV20-fd where=$tmp/bacula-restores select all done yes
257 check_restore_tmp_build_diff