1 package org.eclipse.jgit.internal.storage.dfs;
2
3 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
4 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST;
5 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
6 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
7 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
8 import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
9 import static org.junit.Assert.assertEquals;
10 import static org.junit.Assert.assertFalse;
11 import static org.junit.Assert.assertNotNull;
12 import static org.junit.Assert.assertNull;
13 import static org.junit.Assert.assertSame;
14 import static org.junit.Assert.assertTrue;
15 import static org.junit.Assert.fail;
16
17 import java.io.IOException;
18 import java.util.Collections;
19 import java.util.concurrent.TimeUnit;
20
21 import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
22 import org.eclipse.jgit.internal.storage.reftable.RefCursor;
23 import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
24 import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
25 import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
26 import org.eclipse.jgit.junit.MockSystemReader;
27 import org.eclipse.jgit.junit.TestRepository;
28 import org.eclipse.jgit.lib.AnyObjectId;
29 import org.eclipse.jgit.lib.BatchRefUpdate;
30 import org.eclipse.jgit.lib.NullProgressMonitor;
31 import org.eclipse.jgit.lib.ObjectId;
32 import org.eclipse.jgit.lib.ObjectIdRef;
33 import org.eclipse.jgit.lib.Ref;
34 import org.eclipse.jgit.lib.Repository;
35 import org.eclipse.jgit.revwalk.RevBlob;
36 import org.eclipse.jgit.revwalk.RevCommit;
37 import org.eclipse.jgit.revwalk.RevWalk;
38 import org.eclipse.jgit.storage.pack.PackConfig;
39 import org.eclipse.jgit.transport.ReceiveCommand;
40 import org.eclipse.jgit.util.SystemReader;
41 import org.junit.After;
42 import org.junit.Before;
43 import org.junit.Test;
44
45
46 public class DfsGarbageCollectorTest {
47 private TestRepository<InMemoryRepository> git;
48 private InMemoryRepository repo;
49 private DfsObjDatabase odb;
50 private MockSystemReader mockSystemReader;
51
52 @Before
53 public void setUp() throws IOException {
54 DfsRepositoryDescription desc = new DfsRepositoryDescription("test");
55 git = new TestRepository<>(new InMemoryRepository(desc));
56 repo = git.getRepository();
57 odb = repo.getObjectDatabase();
58 mockSystemReader = new MockSystemReader();
59 SystemReader.setInstance(mockSystemReader);
60 }
61
62 @After
63 public void tearDown() {
64 SystemReader.setInstance(null);
65 }
66
67 @Test
68 public void testCollectionWithNoGarbage() throws Exception {
69 RevCommit commit0 = commit().message("0").create();
70 RevCommit commit1 = commit().message("1").parent(commit0).create();
71 git.update("master", commit1);
72
73 assertTrue("commit0 reachable", isReachable(repo, commit0));
74 assertTrue("commit1 reachable", isReachable(repo, commit1));
75
76
77 assertEquals(2, odb.getPacks().length);
78 for (DfsPackFile pack : odb.getPacks()) {
79 assertEquals(INSERT, pack.getPackDescription().getPackSource());
80 }
81
82 gcNoTtl();
83
84
85 assertEquals(1, odb.getPacks().length);
86 DfsPackFile pack = odb.getPacks()[0];
87 assertEquals(GC, pack.getPackDescription().getPackSource());
88 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
89 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
90 }
91
92 @Test
93 public void testRacyNoReusePrefersSmaller() throws Exception {
94 StringBuilder msg = new StringBuilder();
95 for (int i = 0; i < 100; i++) {
96 msg.append(i).append(": i am a teapot\n");
97 }
98 RevBlob a = git.blob(msg.toString());
99 RevCommit c0 = git.commit()
100 .add("tea", a)
101 .message("0")
102 .create();
103
104 msg.append("short and stout\n");
105 RevBlob b = git.blob(msg.toString());
106 RevCommit c1 = git.commit().parent(c0).tick(1)
107 .add("tea", b)
108 .message("1")
109 .create();
110 git.update("master", c1);
111
112 PackConfig cfg = new PackConfig();
113 cfg.setReuseObjects(false);
114 cfg.setReuseDeltas(false);
115 cfg.setDeltaCompress(false);
116 cfg.setThreads(1);
117 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
118 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
119 gc.setPackConfig(cfg);
120 run(gc);
121
122 assertEquals(1, odb.getPacks().length);
123 DfsPackDescription large = odb.getPacks()[0].getPackDescription();
124 assertSame(PackSource.GC, large.getPackSource());
125
126 cfg.setDeltaCompress(true);
127 gc = new DfsGarbageCollector(repo);
128 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
129 gc.setPackConfig(cfg);
130 run(gc);
131
132 assertEquals(1, odb.getPacks().length);
133 DfsPackDescription small = odb.getPacks()[0].getPackDescription();
134 assertSame(PackSource.GC, small.getPackSource());
135 assertTrue(
136 "delta compression pack is smaller",
137 small.getFileSize(PACK) < large.getFileSize(PACK));
138 assertTrue(
139 "large pack is older",
140 large.getLastModified() < small.getLastModified());
141
142
143 odb.commitPack(Collections.singleton(large), null);
144 odb.clearCache();
145 assertEquals(2, odb.getPacks().length);
146
147 gc = new DfsGarbageCollector(repo);
148 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
149 run(gc);
150
151 assertEquals(1, odb.getPacks().length);
152 DfsPackDescription rebuilt = odb.getPacks()[0].getPackDescription();
153 assertEquals(small.getFileSize(PACK), rebuilt.getFileSize(PACK));
154 }
155
156 @Test
157 public void testCollectionWithGarbage() throws Exception {
158 RevCommit commit0 = commit().message("0").create();
159 RevCommit commit1 = commit().message("1").parent(commit0).create();
160 git.update("master", commit0);
161
162 assertTrue("commit0 reachable", isReachable(repo, commit0));
163 assertFalse("commit1 garbage", isReachable(repo, commit1));
164 gcNoTtl();
165
166 assertEquals(2, odb.getPacks().length);
167 DfsPackFile gc = null;
168 DfsPackFile garbage = null;
169 for (DfsPackFile pack : odb.getPacks()) {
170 DfsPackDescription d = pack.getPackDescription();
171 switch (d.getPackSource()) {
172 case GC:
173 gc = pack;
174 break;
175 case UNREACHABLE_GARBAGE:
176 garbage = pack;
177 break;
178 default:
179 fail("unexpected " + d.getPackSource());
180 break;
181 }
182 }
183
184 assertNotNull("created GC pack", gc);
185 assertTrue(isObjectInPack(commit0, gc));
186
187 assertNotNull("created UNREACHABLE_GARBAGE pack", garbage);
188 assertTrue(isObjectInPack(commit1, garbage));
189 }
190
191 @Test
192 public void testCollectionWithGarbageAndGarbagePacksPurged()
193 throws Exception {
194 RevCommit commit0 = commit().message("0").create();
195 RevCommit commit1 = commit().message("1").parent(commit0).create();
196 git.update("master", commit0);
197
198 gcWithTtl();
199
200
201 assertEquals(2, odb.getPacks().length);
202 boolean gcPackFound = false;
203 boolean garbagePackFound = false;
204 for (DfsPackFile pack : odb.getPacks()) {
205 DfsPackDescription d = pack.getPackDescription();
206 switch (d.getPackSource()) {
207 case GC:
208 gcPackFound = true;
209 assertTrue("has commit0", isObjectInPack(commit0, pack));
210 assertFalse("no commit1", isObjectInPack(commit1, pack));
211 break;
212 case UNREACHABLE_GARBAGE:
213 garbagePackFound = true;
214 assertFalse("no commit0", isObjectInPack(commit0, pack));
215 assertTrue("has commit1", isObjectInPack(commit1, pack));
216 break;
217 default:
218 fail("unexpected " + d.getPackSource());
219 break;
220 }
221 }
222 assertTrue("gc pack found", gcPackFound);
223 assertTrue("garbage pack found", garbagePackFound);
224
225 gcWithTtl();
226
227 DfsPackFile[] packs = odb.getPacks();
228 assertEquals(1, packs.length);
229
230 assertEquals(GC, packs[0].getPackDescription().getPackSource());
231 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
232 assertFalse("no commit1", isObjectInPack(commit1, packs[0]));
233 }
234
235 @Test
236 public void testCollectionWithGarbageAndRereferencingGarbage()
237 throws Exception {
238 RevCommit commit0 = commit().message("0").create();
239 RevCommit commit1 = commit().message("1").parent(commit0).create();
240 git.update("master", commit0);
241
242 gcWithTtl();
243
244
245 assertEquals(2, odb.getPacks().length);
246 boolean gcPackFound = false;
247 boolean garbagePackFound = false;
248 for (DfsPackFile pack : odb.getPacks()) {
249 DfsPackDescription d = pack.getPackDescription();
250 switch (d.getPackSource()) {
251 case GC:
252 gcPackFound = true;
253 assertTrue("has commit0", isObjectInPack(commit0, pack));
254 assertFalse("no commit1", isObjectInPack(commit1, pack));
255 break;
256 case UNREACHABLE_GARBAGE:
257 garbagePackFound = true;
258 assertFalse("no commit0", isObjectInPack(commit0, pack));
259 assertTrue("has commit1", isObjectInPack(commit1, pack));
260 break;
261 default:
262 fail("unexpected " + d.getPackSource());
263 break;
264 }
265 }
266 assertTrue("gc pack found", gcPackFound);
267 assertTrue("garbage pack found", garbagePackFound);
268
269 git.update("master", commit1);
270
271 gcWithTtl();
272
273
274 DfsPackFile[] packs = odb.getPacks();
275 assertEquals(1, packs.length);
276
277 assertEquals(GC, packs[0].getPackDescription().getPackSource());
278 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
279 assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
280 }
281
282 @Test
283 public void testCollectionWithPureGarbageAndGarbagePacksPurged()
284 throws Exception {
285 RevCommit commit0 = commit().message("0").create();
286 RevCommit commit1 = commit().message("1").parent(commit0).create();
287
288 gcWithTtl();
289
290
291 DfsPackFile[] packs = odb.getPacks();
292 assertEquals(1, packs.length);
293
294 assertEquals(UNREACHABLE_GARBAGE, packs[0].getPackDescription().getPackSource());
295 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
296 assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
297
298 gcWithTtl();
299
300
301 assertEquals(0, odb.getPacks().length);
302 }
303
304 @Test
305 public void testCollectionWithPureGarbageAndRereferencingGarbage()
306 throws Exception {
307 RevCommit commit0 = commit().message("0").create();
308 RevCommit commit1 = commit().message("1").parent(commit0).create();
309
310 gcWithTtl();
311
312
313 DfsPackFile[] packs = odb.getPacks();
314 assertEquals(1, packs.length);
315
316 DfsPackDescription pack = packs[0].getPackDescription();
317 assertEquals(UNREACHABLE_GARBAGE, pack.getPackSource());
318 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
319 assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
320
321 git.update("master", commit0);
322
323 gcWithTtl();
324
325
326 packs = odb.getPacks();
327 assertEquals(1, packs.length);
328
329 pack = packs[0].getPackDescription();
330 assertEquals(GC, pack.getPackSource());
331 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
332 assertFalse("no commit1", isObjectInPack(commit1, packs[0]));
333 }
334
335 @Test
336 public void testCollectionWithGarbageCoalescence() throws Exception {
337 RevCommit commit0 = commit().message("0").create();
338 RevCommit commit1 = commit().message("1").parent(commit0).create();
339 git.update("master", commit0);
340
341 for (int i = 0; i < 3; i++) {
342 commit1 = commit().message("g" + i).parent(commit1).create();
343
344
345
346 gcNoTtl();
347 assertEquals(1, countPacks(UNREACHABLE_GARBAGE));
348 }
349 }
350
351 @Test
352 public void testCollectionWithGarbageNoCoalescence() throws Exception {
353 RevCommit commit0 = commit().message("0").create();
354 RevCommit commit1 = commit().message("1").parent(commit0).create();
355 git.update("master", commit0);
356
357 for (int i = 0; i < 3; i++) {
358 commit1 = commit().message("g" + i).parent(commit1).create();
359
360 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
361 gc.setCoalesceGarbageLimit(0);
362 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
363 run(gc);
364 assertEquals(1 + i, countPacks(UNREACHABLE_GARBAGE));
365 }
366 }
367
368 @Test
369 public void testCollectionWithGarbageCoalescenceWithShortTtl()
370 throws Exception {
371 RevCommit commit0 = commit().message("0").create();
372 RevCommit commit1 = commit().message("1").parent(commit0).create();
373 git.update("master", commit0);
374
375
376 for (int i = 0; i < 100; i++) {
377 mockSystemReader.tick(60);
378 commit1 = commit().message("g" + i).parent(commit1).create();
379
380 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
381 gc.setGarbageTtl(1, TimeUnit.HOURS);
382 run(gc);
383
384
385
386
387
388 int count = countPacks(UNREACHABLE_GARBAGE);
389 assertTrue("Garbage pack count should not exceed 4, but found "
390 + count, count <= 4);
391 }
392 }
393
394 @Test
395 public void testCollectionWithGarbageCoalescenceWithLongTtl()
396 throws Exception {
397 RevCommit commit0 = commit().message("0").create();
398 RevCommit commit1 = commit().message("1").parent(commit0).create();
399 git.update("master", commit0);
400
401
402 for (int i = 0; i < 100; i++) {
403 mockSystemReader.tick(3600);
404 commit1 = commit().message("g" + i).parent(commit1).create();
405
406 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
407 gc.setGarbageTtl(2, TimeUnit.DAYS);
408 run(gc);
409
410
411
412
413
414 int count = countPacks(UNREACHABLE_GARBAGE);
415 assertTrue("Garbage pack count should not exceed 3, but found "
416 + count, count <= 3);
417 }
418 }
419
420 @Test
421 public void testEstimateGcPackSizeInNewRepo() throws Exception {
422 RevCommit commit0 = commit().message("0").create();
423 RevCommit commit1 = commit().message("1").parent(commit0).create();
424 git.update("master", commit1);
425
426
427 long inputPacksSize = 32;
428 assertEquals(2, odb.getPacks().length);
429 for (DfsPackFile pack : odb.getPacks()) {
430 assertEquals(INSERT, pack.getPackDescription().getPackSource());
431 inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
432 }
433
434 gcNoTtl();
435
436
437 assertEquals(1, odb.getPacks().length);
438 DfsPackFile pack = odb.getPacks()[0];
439 assertEquals(GC, pack.getPackDescription().getPackSource());
440 assertEquals(inputPacksSize,
441 pack.getPackDescription().getEstimatedPackSize());
442 }
443
444 @Test
445 public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception {
446 RevCommit commit0 = commit().message("0").create();
447 RevCommit commit1 = commit().message("1").parent(commit0).create();
448 git.update("master", commit1);
449
450 gcNoTtl();
451
452 RevCommit commit2 = commit().message("2").parent(commit1).create();
453 git.update("master", commit2);
454
455
456 assertEquals(2, odb.getPacks().length);
457 boolean gcPackFound = false;
458 boolean insertPackFound = false;
459 long inputPacksSize = 32;
460 for (DfsPackFile pack : odb.getPacks()) {
461 DfsPackDescription d = pack.getPackDescription();
462 switch (d.getPackSource()) {
463 case GC:
464 gcPackFound = true;
465 break;
466 case INSERT:
467 insertPackFound = true;
468 break;
469 default:
470 fail("unexpected " + d.getPackSource());
471 break;
472 }
473 inputPacksSize += d.getFileSize(PACK) - 32;
474 }
475 assertTrue(gcPackFound);
476 assertTrue(insertPackFound);
477
478 gcNoTtl();
479
480
481 DfsPackFile pack = odb.getPacks()[0];
482 assertEquals(GC, pack.getPackDescription().getPackSource());
483 assertEquals(inputPacksSize,
484 pack.getPackDescription().getEstimatedPackSize());
485 }
486
487 @Test
488 public void testEstimateGcRestPackSizeInNewRepo() throws Exception {
489 RevCommit commit0 = commit().message("0").create();
490 RevCommit commit1 = commit().message("1").parent(commit0).create();
491 git.update("refs/notes/note1", commit1);
492
493
494 long inputPacksSize = 32;
495 assertEquals(2, odb.getPacks().length);
496 for (DfsPackFile pack : odb.getPacks()) {
497 assertEquals(INSERT, pack.getPackDescription().getPackSource());
498 inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
499 }
500
501 gcNoTtl();
502
503
504 assertEquals(1, odb.getPacks().length);
505 DfsPackFile pack = odb.getPacks()[0];
506 assertEquals(GC_REST, pack.getPackDescription().getPackSource());
507 assertEquals(inputPacksSize,
508 pack.getPackDescription().getEstimatedPackSize());
509 }
510
511 @Test
512 public void testEstimateGcRestPackSizeWithAnExistingGcPack()
513 throws Exception {
514 RevCommit commit0 = commit().message("0").create();
515 RevCommit commit1 = commit().message("1").parent(commit0).create();
516 git.update("refs/notes/note1", commit1);
517
518 gcNoTtl();
519
520 RevCommit commit2 = commit().message("2").parent(commit1).create();
521 git.update("refs/notes/note2", commit2);
522
523
524 assertEquals(2, odb.getPacks().length);
525 boolean gcRestPackFound = false;
526 boolean insertPackFound = false;
527 long inputPacksSize = 32;
528 for (DfsPackFile pack : odb.getPacks()) {
529 DfsPackDescription d = pack.getPackDescription();
530 switch (d.getPackSource()) {
531 case GC_REST:
532 gcRestPackFound = true;
533 break;
534 case INSERT:
535 insertPackFound = true;
536 break;
537 default:
538 fail("unexpected " + d.getPackSource());
539 break;
540 }
541 inputPacksSize += d.getFileSize(PACK) - 32;
542 }
543 assertTrue(gcRestPackFound);
544 assertTrue(insertPackFound);
545
546 gcNoTtl();
547
548
549 DfsPackFile pack = odb.getPacks()[0];
550 assertEquals(GC_REST, pack.getPackDescription().getPackSource());
551 assertEquals(inputPacksSize,
552 pack.getPackDescription().getEstimatedPackSize());
553 }
554
555 @Test
556 public void testEstimateGcPackSizesWithGcAndGcRestPacks() throws Exception {
557 RevCommit commit0 = commit().message("0").create();
558 git.update("head", commit0);
559 RevCommit commit1 = commit().message("1").parent(commit0).create();
560 git.update("refs/notes/note1", commit1);
561
562 gcNoTtl();
563
564 RevCommit commit2 = commit().message("2").parent(commit1).create();
565 git.update("refs/notes/note2", commit2);
566
567
568 assertEquals(3, odb.getPacks().length);
569 boolean gcPackFound = false;
570 boolean gcRestPackFound = false;
571 boolean insertPackFound = false;
572 long gcPackSize = 0;
573 long gcRestPackSize = 0;
574 long insertPackSize = 0;
575 for (DfsPackFile pack : odb.getPacks()) {
576 DfsPackDescription d = pack.getPackDescription();
577 switch (d.getPackSource()) {
578 case GC:
579 gcPackFound = true;
580 gcPackSize = d.getFileSize(PACK);
581 break;
582 case GC_REST:
583 gcRestPackFound = true;
584 gcRestPackSize = d.getFileSize(PACK);
585 break;
586 case INSERT:
587 insertPackFound = true;
588 insertPackSize = d.getFileSize(PACK);
589 break;
590 default:
591 fail("unexpected " + d.getPackSource());
592 break;
593 }
594 }
595 assertTrue(gcPackFound);
596 assertTrue(gcRestPackFound);
597 assertTrue(insertPackFound);
598
599 gcNoTtl();
600
601
602
603
604
605 assertEquals(2, odb.getPacks().length);
606 gcPackFound = false;
607 gcRestPackFound = false;
608 for (DfsPackFile pack : odb.getPacks()) {
609 DfsPackDescription d = pack.getPackDescription();
610 switch (d.getPackSource()) {
611 case GC:
612 gcPackFound = true;
613 assertEquals(gcPackSize + insertPackSize - 32,
614 pack.getPackDescription().getEstimatedPackSize());
615 break;
616 case GC_REST:
617 gcRestPackFound = true;
618 assertEquals(gcRestPackSize + insertPackSize - 32,
619 pack.getPackDescription().getEstimatedPackSize());
620 break;
621 default:
622 fail("unexpected " + d.getPackSource());
623 break;
624 }
625 }
626 assertTrue(gcPackFound);
627 assertTrue(gcRestPackFound);
628 }
629
630 @Test
631 public void testEstimateUnreachableGarbagePackSize() throws Exception {
632 RevCommit commit0 = commit().message("0").create();
633 RevCommit commit1 = commit().message("1").parent(commit0).create();
634 git.update("master", commit0);
635
636 assertTrue("commit0 reachable", isReachable(repo, commit0));
637 assertFalse("commit1 garbage", isReachable(repo, commit1));
638
639
640 long packSize0 = 0;
641 long packSize1 = 0;
642 assertEquals(2, odb.getPacks().length);
643 for (DfsPackFile pack : odb.getPacks()) {
644 DfsPackDescription d = pack.getPackDescription();
645 assertEquals(INSERT, d.getPackSource());
646 if (isObjectInPack(commit0, pack)) {
647 packSize0 = d.getFileSize(PACK);
648 } else if (isObjectInPack(commit1, pack)) {
649 packSize1 = d.getFileSize(PACK);
650 } else {
651 fail("expected object not found in the pack");
652 }
653 }
654
655 gcNoTtl();
656
657 assertEquals(2, odb.getPacks().length);
658 for (DfsPackFile pack : odb.getPacks()) {
659 DfsPackDescription d = pack.getPackDescription();
660 switch (d.getPackSource()) {
661 case GC:
662
663
664
665
666 assertEquals(packSize0 + packSize1 - 32,
667 d.getEstimatedPackSize());
668 break;
669 case UNREACHABLE_GARBAGE:
670
671 assertEquals(packSize1, d.getEstimatedPackSize());
672 break;
673 default:
674 fail("unexpected " + d.getPackSource());
675 break;
676 }
677 }
678 }
679
680 @Test
681 public void testSinglePackForAllRefs() throws Exception {
682 RevCommit commit0 = commit().message("0").create();
683 git.update("head", commit0);
684 RevCommit commit1 = commit().message("1").parent(commit0).create();
685 git.update("refs/notes/note1", commit1);
686
687 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
688 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
689 gc.getPackConfig().setSinglePack(true);
690 run(gc);
691 assertEquals(1, odb.getPacks().length);
692
693 gc = new DfsGarbageCollector(repo);
694 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
695 gc.getPackConfig().setSinglePack(false);
696 run(gc);
697 assertEquals(2, odb.getPacks().length);
698 }
699
700 @SuppressWarnings("boxing")
701 @Test
702 public void producesNewReftable() throws Exception {
703 String master = "refs/heads/master";
704 RevCommit commit0 = commit().message("0").create();
705 RevCommit commit1 = commit().message("1").parent(commit0).create();
706
707 BatchRefUpdate bru = git.getRepository().getRefDatabase()
708 .newBatchUpdate();
709 bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
710 for (int i = 1; i <= 5100; i++) {
711 bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
712 String.format("refs/pulls/%04d", i)));
713 }
714 try (RevWalk rw = new RevWalk(git.getRepository())) {
715 bru.execute(rw, NullProgressMonitor.INSTANCE);
716 }
717
718 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
719 gc.setReftableConfig(new ReftableConfig());
720 run(gc);
721
722
723 assertEquals(1, odb.getPacks().length);
724 DfsPackFile pack = odb.getPacks()[0];
725 DfsPackDescription desc = pack.getPackDescription();
726 assertEquals(GC, desc.getPackSource());
727 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
728 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
729
730
731 assertTrue(desc.hasFileExt(REFTABLE));
732 ReftableWriter.Stats stats = desc.getReftableStats();
733 assertNotNull(stats);
734 assertTrue(stats.totalBytes() > 0);
735 assertEquals(5101, stats.refCount());
736 assertEquals(1, stats.minUpdateIndex());
737 assertEquals(1, stats.maxUpdateIndex());
738
739 DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
740 try (DfsReader ctx = odb.newReader();
741 ReftableReader rr = table.open(ctx);
742 RefCursor rc = rr.seekRef("refs/pulls/5100")) {
743 assertTrue(rc.next());
744 assertEquals(commit0, rc.getRef().getObjectId());
745 assertFalse(rc.next());
746 }
747 }
748
749 @Test
750 public void leavesNonGcReftablesIfNotConfigured() throws Exception {
751 String master = "refs/heads/master";
752 RevCommit commit0 = commit().message("0").create();
753 RevCommit commit1 = commit().message("1").parent(commit0).create();
754 git.update(master, commit1);
755
756 DfsPackDescription t1 = odb.newPack(INSERT);
757 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
758 new ReftableWriter(out).begin().finish();
759 t1.addFileExt(REFTABLE);
760 }
761 odb.commitPack(Collections.singleton(t1), null);
762
763 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
764 gc.setReftableConfig(null);
765 run(gc);
766
767
768 assertEquals(1, odb.getPacks().length);
769 DfsPackFile pack = odb.getPacks()[0];
770 DfsPackDescription desc = pack.getPackDescription();
771 assertEquals(GC, desc.getPackSource());
772 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
773 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
774
775
776 DfsReftable[] tables = odb.getReftables();
777 assertEquals(2, tables.length);
778 assertEquals(t1, tables[0].getPackDescription());
779 }
780
781 @Test
782 public void prunesNonGcReftables() throws Exception {
783 String master = "refs/heads/master";
784 RevCommit commit0 = commit().message("0").create();
785 RevCommit commit1 = commit().message("1").parent(commit0).create();
786 git.update(master, commit1);
787
788 DfsPackDescription t1 = odb.newPack(INSERT);
789 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
790 new ReftableWriter(out).begin().finish();
791 t1.addFileExt(REFTABLE);
792 }
793 odb.commitPack(Collections.singleton(t1), null);
794 odb.clearCache();
795
796 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
797 gc.setReftableConfig(new ReftableConfig());
798 run(gc);
799
800
801 assertEquals(1, odb.getPacks().length);
802 DfsPackFile pack = odb.getPacks()[0];
803 DfsPackDescription desc = pack.getPackDescription();
804 assertEquals(GC, desc.getPackSource());
805 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
806 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
807
808
809 DfsReftable[] tables = odb.getReftables();
810 assertEquals(1, tables.length);
811 assertEquals(desc, tables[0].getPackDescription());
812 assertTrue(desc.hasFileExt(REFTABLE));
813 }
814
815 @Test
816 public void compactsReftables() throws Exception {
817 String master = "refs/heads/master";
818 RevCommit commit0 = commit().message("0").create();
819 RevCommit commit1 = commit().message("1").parent(commit0).create();
820 git.update(master, commit1);
821
822 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
823 gc.setReftableConfig(new ReftableConfig());
824 run(gc);
825
826 DfsPackDescription t1 = odb.newPack(INSERT);
827 Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
828 "refs/heads/next", commit0.copy());
829 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
830 ReftableWriter w = new ReftableWriter(out);
831 w.setMinUpdateIndex(42);
832 w.setMaxUpdateIndex(42);
833 w.begin();
834 w.sortAndWriteRefs(Collections.singleton(next));
835 w.finish();
836 t1.addFileExt(REFTABLE);
837 t1.setReftableStats(w.getStats());
838 }
839 odb.commitPack(Collections.singleton(t1), null);
840
841 gc = new DfsGarbageCollector(repo);
842 gc.setReftableConfig(new ReftableConfig());
843 run(gc);
844
845
846 assertEquals(1, odb.getPacks().length);
847 DfsPackFile pack = odb.getPacks()[0];
848 DfsPackDescription desc = pack.getPackDescription();
849 assertEquals(GC, desc.getPackSource());
850 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
851 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
852
853
854 DfsReftable[] tables = odb.getReftables();
855 assertEquals(1, tables.length);
856 assertEquals(desc, tables[0].getPackDescription());
857 assertTrue(desc.hasFileExt(REFTABLE));
858
859
860 DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
861 try (DfsReader ctx = odb.newReader();
862 ReftableReader rr = table.open(ctx);
863 RefCursor rc = rr.allRefs()) {
864 assertEquals(1, rr.minUpdateIndex());
865 assertEquals(42, rr.maxUpdateIndex());
866
867 assertTrue(rc.next());
868 assertEquals(master, rc.getRef().getName());
869 assertEquals(commit1, rc.getRef().getObjectId());
870
871 assertTrue(rc.next());
872 assertEquals(next.getName(), rc.getRef().getName());
873 assertEquals(commit0, rc.getRef().getObjectId());
874
875 assertFalse(rc.next());
876 }
877 }
878
879 @Test
880 public void reftableWithoutTombstoneResurrected() throws Exception {
881 RevCommit commit0 = commit().message("0").create();
882 String NEXT = "refs/heads/next";
883 DfsRefDatabase refdb = (DfsRefDatabase)repo.getRefDatabase();
884 git.update(NEXT, commit0);
885 Ref next = refdb.exactRef(NEXT);
886 assertNotNull(next);
887 assertEquals(commit0, next.getObjectId());
888
889 git.delete(NEXT);
890 refdb.clearCache();
891 assertNull(refdb.exactRef(NEXT));
892
893 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
894 gc.setReftableConfig(new ReftableConfig());
895 gc.setIncludeDeletes(false);
896 gc.setConvertToReftable(false);
897 run(gc);
898 assertEquals(1, odb.getReftables().length);
899 try (DfsReader ctx = odb.newReader();
900 ReftableReader rr = odb.getReftables()[0].open(ctx)) {
901 rr.setIncludeDeletes(true);
902 assertEquals(1, rr.minUpdateIndex());
903 assertEquals(2, rr.maxUpdateIndex());
904 assertNull(rr.exactRef(NEXT));
905 }
906
907 RevCommit commit1 = commit().message("1").create();
908 DfsPackDescription t1 = odb.newPack(INSERT);
909 Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT,
910 commit1);
911 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
912 ReftableWriter w = new ReftableWriter(out)
913 .setMinUpdateIndex(1)
914 .setMaxUpdateIndex(1)
915 .begin();
916 w.writeRef(newNext, 1);
917 w.finish();
918 t1.addFileExt(REFTABLE);
919 t1.setReftableStats(w.getStats());
920 }
921 odb.commitPack(Collections.singleton(t1), null);
922 assertEquals(2, odb.getReftables().length);
923 refdb.clearCache();
924 newNext = refdb.exactRef(NEXT);
925 assertNotNull(newNext);
926 assertEquals(commit1, newNext.getObjectId());
927 }
928
929 @Test
930 public void reftableWithTombstoneNotResurrected() throws Exception {
931 RevCommit commit0 = commit().message("0").create();
932 String NEXT = "refs/heads/next";
933 DfsRefDatabase refdb = (DfsRefDatabase)repo.getRefDatabase();
934 git.update(NEXT, commit0);
935 Ref next = refdb.exactRef(NEXT);
936 assertNotNull(next);
937 assertEquals(commit0, next.getObjectId());
938
939 git.delete(NEXT);
940 refdb.clearCache();
941 assertNull(refdb.exactRef(NEXT));
942
943 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
944 gc.setReftableConfig(new ReftableConfig());
945 gc.setIncludeDeletes(true);
946 gc.setConvertToReftable(false);
947 run(gc);
948 assertEquals(1, odb.getReftables().length);
949 try (DfsReader ctx = odb.newReader();
950 ReftableReader rr = odb.getReftables()[0].open(ctx)) {
951 rr.setIncludeDeletes(true);
952 assertEquals(1, rr.minUpdateIndex());
953 assertEquals(2, rr.maxUpdateIndex());
954 next = rr.exactRef(NEXT);
955 assertNotNull(next);
956 assertNull(next.getObjectId());
957 }
958
959 RevCommit commit1 = commit().message("1").create();
960 DfsPackDescription t1 = odb.newPack(INSERT);
961 Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT,
962 commit1);
963 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
964 ReftableWriter w = new ReftableWriter(out)
965 .setMinUpdateIndex(1)
966 .setMaxUpdateIndex(1)
967 .begin();
968 w.writeRef(newNext, 1);
969 w.finish();
970 t1.addFileExt(REFTABLE);
971 t1.setReftableStats(w.getStats());
972 }
973 odb.commitPack(Collections.singleton(t1), null);
974 assertEquals(2, odb.getReftables().length);
975 refdb.clearCache();
976 assertNull(refdb.exactRef(NEXT));
977 }
978
979 private TestRepository<InMemoryRepository>.CommitBuilder commit() {
980 return git.commit();
981 }
982
983 private void gcNoTtl() throws IOException {
984 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
985 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
986 run(gc);
987 }
988
989 private void gcWithTtl() throws IOException {
990
991 mockSystemReader.tick(60);
992 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
993 gc.setGarbageTtl(1, TimeUnit.MINUTES);
994 run(gc);
995 }
996
997 private void run(DfsGarbageCollector gc) throws IOException {
998
999 mockSystemReader.tick(1);
1000 assertTrue("gc repacked", gc.pack(null));
1001 odb.clearCache();
1002 }
1003
1004 private static boolean isReachable(Repository repo, AnyObjectId id)
1005 throws IOException {
1006 try (RevWalk rw = new RevWalk(repo)) {
1007 for (Ref ref : repo.getRefDatabase().getRefs()) {
1008 rw.markStart(rw.parseCommit(ref.getObjectId()));
1009 }
1010 for (RevCommit next; (next = rw.next()) != null;) {
1011 if (AnyObjectId.isEqual(next, id)) {
1012 return true;
1013 }
1014 }
1015 }
1016 return false;
1017 }
1018
1019 private boolean isObjectInPack(AnyObjectId id, DfsPackFile pack)
1020 throws IOException {
1021 try (DfsReader reader = odb.newReader()) {
1022 return pack.hasObject(reader, id);
1023 }
1024 }
1025
1026 private int countPacks(PackSource source) throws IOException {
1027 int cnt = 0;
1028 for (DfsPackFile pack : odb.getPacks()) {
1029 if (pack.getPackDescription().getPackSource() == source) {
1030 cnt++;
1031 }
1032 }
1033 return cnt;
1034 }
1035 }