GC.java

  1. /*
  2.  * Copyright (C) 2012, Christian Halstrick <christian.halstrick@sap.com>
  3.  * Copyright (C) 2011, Shawn O. Pearce <spearce@spearce.org> and others
  4.  *
  5.  * This program and the accompanying materials are made available under the
  6.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  7.  * https://www.eclipse.org/org/documents/edl-v10.php.
  8.  *
  9.  * SPDX-License-Identifier: BSD-3-Clause
  10.  */
  11. package org.eclipse.jgit.internal.storage.file;

  13. import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
  14. import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
  15. import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
  16. import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;

  18. import java.io.File;
  19. import java.io.FileOutputStream;
  20. import java.io.IOException;
  21. import java.io.OutputStream;
  22. import java.io.PrintWriter;
  23. import java.io.StringWriter;
  24. import java.nio.channels.Channels;
  25. import java.nio.channels.FileChannel;
  26. import java.nio.file.DirectoryNotEmptyException;
  27. import java.nio.file.DirectoryStream;
  28. import java.nio.file.Files;
  29. import java.nio.file.Path;
  30. import java.nio.file.StandardCopyOption;
  31. import java.text.MessageFormat;
  32. import java.text.ParseException;
  33. import java.time.Instant;
  34. import java.time.temporal.ChronoUnit;
  35. import java.util.ArrayList;
  36. import java.util.Collection;
  37. import java.util.Collections;
  38. import java.util.Comparator;
  39. import java.util.Date;
  40. import java.util.HashMap;
  41. import java.util.HashSet;
  42. import java.util.Iterator;
  43. import java.util.LinkedList;
  44. import java.util.List;
  45. import java.util.Map;
  46. import java.util.Objects;
  47. import java.util.Set;
  48. import java.util.TreeMap;
  49. import java.util.concurrent.CompletableFuture;
  50. import java.util.concurrent.ExecutorService;
  51. import java.util.function.Supplier;
  52. import java.util.regex.Pattern;
  53. import java.util.stream.Collectors;
  54. import java.util.stream.Stream;

  56. import org.eclipse.jgit.annotations.NonNull;
  57. import org.eclipse.jgit.dircache.DirCacheIterator;
  58. import org.eclipse.jgit.errors.CancelledException;
  59. import org.eclipse.jgit.errors.CorruptObjectException;
  60. import org.eclipse.jgit.errors.IncorrectObjectTypeException;
  61. import org.eclipse.jgit.errors.MissingObjectException;
  62. import org.eclipse.jgit.errors.NoWorkTreeException;
  63. import org.eclipse.jgit.internal.JGitText;
  64. import org.eclipse.jgit.internal.storage.pack.PackExt;
  65. import org.eclipse.jgit.internal.storage.pack.PackWriter;
  66. import org.eclipse.jgit.lib.ConfigConstants;
  67. import org.eclipse.jgit.lib.Constants;
  68. import org.eclipse.jgit.lib.FileMode;
  69. import org.eclipse.jgit.lib.NullProgressMonitor;
  70. import org.eclipse.jgit.lib.ObjectId;
  71. import org.eclipse.jgit.lib.ObjectIdSet;
  72. import org.eclipse.jgit.lib.ObjectLoader;
  73. import org.eclipse.jgit.lib.ObjectReader;
  74. import org.eclipse.jgit.lib.ProgressMonitor;
  75. import org.eclipse.jgit.lib.Ref;
  76. import org.eclipse.jgit.lib.Ref.Storage;
  77. import org.eclipse.jgit.lib.RefDatabase;
  78. import org.eclipse.jgit.lib.ReflogEntry;
  79. import org.eclipse.jgit.lib.ReflogReader;
  80. import org.eclipse.jgit.lib.internal.WorkQueue;
  81. import org.eclipse.jgit.revwalk.ObjectWalk;
  82. import org.eclipse.jgit.revwalk.RevObject;
  83. import org.eclipse.jgit.revwalk.RevWalk;
  84. import org.eclipse.jgit.storage.pack.PackConfig;
  85. import org.eclipse.jgit.treewalk.TreeWalk;
  86. import org.eclipse.jgit.treewalk.filter.TreeFilter;
  87. import org.eclipse.jgit.util.FileUtils;
  88. import org.eclipse.jgit.util.GitDateParser;
  89. import org.eclipse.jgit.util.SystemReader;
  90. import org.slf4j.Logger;
  91. import org.slf4j.LoggerFactory;

  93. /**
  94.  * A garbage collector for git
  95.  * {@link org.eclipse.jgit.internal.storage.file.FileRepository}. Instances of
  96.  * this class are not thread-safe. Don't use the same instance from multiple
  97.  * threads.
  98.  *
  99.  * This class started as a copy of DfsGarbageCollector from Shawn O. Pearce
  100.  * adapted to FileRepositories.
  101.  */
  102. public class GC {
  103.     private static final Logger LOG = LoggerFactory
  104.             .getLogger(GC.class);

  106.     private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$

  108.     private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$

  110.     private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
  111.             .compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$

  113.     private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$

  115.     private static final String BITMAP_EXT = "." //$NON-NLS-1$
  116.             + PackExt.BITMAP_INDEX.getExtension();

  118.     private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$

  120.     private static final String KEEP_EXT = "." + PackExt.KEEP.getExtension(); //$NON-NLS-1$

  122.     private static final int DEFAULT_AUTOPACKLIMIT = 50;

  124.     private static final int DEFAULT_AUTOLIMIT = 6700;

  126.     private static volatile ExecutorService executor;

  128.     /**
  129.      * Set the executor for running auto-gc in the background. If no executor is
  130.      * set JGit's own WorkQueue will be used.
  131.      *
  132.      * @param e
  133.      *            the executor to be used for running auto-gc
  134.      */
  135.     public static void setExecutor(ExecutorService e) {
  136.         executor = e;
  137.     }

  139.     private final FileRepository repo;

  141.     private ProgressMonitor pm;

  143.     private long expireAgeMillis = -1;

  145.     private Date expire;

  147.     private long packExpireAgeMillis = -1;

  149.     private Date packExpire;

  151.     private PackConfig pconfig;

  153.     /**
  154.      * the refs which existed during the last call to {@link #repack()}. This is
  155.      * needed during {@link #prune(Set)} where we can optimize by looking at the
  156.      * difference between the current refs and the refs which existed during
  157.      * last {@link #repack()}.
  158.      */
  159.     private Collection<Ref> lastPackedRefs;

  161.     /**
  162.      * Holds the starting time of the last repack() execution. This is needed in
  163.      * prune() to inspect only those reflog entries which have been added since
  164.      * last repack().
  165.      */
  166.     private long lastRepackTime;

  168.     /**
  169.      * Whether gc should do automatic housekeeping
  170.      */
  171.     private boolean automatic;

  173.     /**
  174.      * Whether to run gc in a background thread
  175.      */
  176.     private boolean background;

  178.     /**
  179.      * Creates a new garbage collector with default values. An expirationTime of
  180.      * two weeks and <code>null</code> as progress monitor will be used.
  181.      *
  182.      * @param repo
  183.      *            the repo to work on
  184.      */
  185.     public GC(FileRepository repo) {
  186.         this.repo = repo;
  187.         this.pconfig = new PackConfig(repo);
  188.         this.pm = NullProgressMonitor.INSTANCE;
  189.     }

  191.     /**
  192.      * Runs a garbage collector on a
  193.      * {@link org.eclipse.jgit.internal.storage.file.FileRepository}. It will
  194.      * <ul>
  195.      * <li>pack loose references into packed-refs</li>
  196.      * <li>repack all reachable objects into new pack files and delete the old
  197.      * pack files</li>
  198.      * <li>prune all loose objects which are now reachable by packs</li>
  199.      * </ul>
  200.      *
  201.      * If {@link #setAuto(boolean)} was set to {@code true} {@code gc} will
  202.      * first check whether any housekeeping is required; if not, it exits
  203.      * without performing any work.
  204.      *
  205.      * If {@link #setBackground(boolean)} was set to {@code true}
  206.      * {@code collectGarbage} will start the gc in the background, and then
  207.      * return immediately. In this case, errors will not be reported except in
  208.      * gc.log.
  209.      *
  210.      * @return the collection of
  211.      *         {@link org.eclipse.jgit.internal.storage.file.Pack}'s which
  212.      *         are newly created
  213.      * @throws java.io.IOException
  214.      * @throws java.text.ParseException
  215.      *             If the configuration parameter "gc.pruneexpire" couldn't be
  216.      *             parsed
  217.      */
  218.     public CompletableFuture<Collection<Pack>> gc()
  219.             throws IOException, ParseException {
  220.         if (!background) {
  221.             return CompletableFuture.completedFuture(doGc());
  222.         }
  223.         final GcLog gcLog = new GcLog(repo);
  224.         if (!gcLog.lock()) {
  225.             // there is already a background gc running
  226.             return CompletableFuture.completedFuture(Collections.emptyList());
  227.         }

  229.         Supplier<Collection<Pack>> gcTask = () -> {
  230.             try {
  231.                 Collection<Pack> newPacks = doGc();
  232.                 if (automatic && tooManyLooseObjects()) {
  233.                     String message = JGitText.get().gcTooManyUnpruned;
  234.                     gcLog.write(message);
  235.                     gcLog.commit();
  236.                 }
  237.                 return newPacks;
  238.             } catch (IOException | ParseException e) {
  239.                 try {
  240.                     gcLog.write(e.getMessage());
  241.                     StringWriter sw = new StringWriter();
  242.                     e.printStackTrace(new PrintWriter(sw));
  243.                     gcLog.write(sw.toString());
  244.                     gcLog.commit();
  245.                 } catch (IOException e2) {
  246.                     e2.addSuppressed(e);
  247.                     LOG.error(e2.getMessage(), e2);
  248.                 }
  249.             } finally {
  250.                 gcLog.unlock();
  251.             }
  252.             return Collections.emptyList();
  253.         };
  254.         return CompletableFuture.supplyAsync(gcTask, executor());
  255.     }

  257.     private ExecutorService executor() {
  258.         return (executor != null) ? executor : WorkQueue.getExecutor();
  259.     }

  261.     private Collection<Pack> doGc() throws IOException, ParseException {
  262.         if (automatic && !needGc()) {
  263.             return Collections.emptyList();
  264.         }
  265.         pm.start(6 /* tasks */);
  266.         packRefs();
  267.         // TODO: implement reflog_expire(pm, repo);
  268.         Collection<Pack> newPacks = repack();
  269.         prune(Collections.emptySet());
  270.         // TODO: implement rerere_gc(pm);
  271.         return newPacks;
  272.     }

  274.     /**
  275.      * Loosen objects in a pack file which are not also in the newly-created
  276.      * pack files.
  277.      *
  278.      * @param inserter
  279.      * @param reader
  280.      * @param pack
  281.      * @param existing
  282.      * @throws IOException
  283.      */
  284.     private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, Pack pack, HashSet<ObjectId> existing)
  285.             throws IOException {
  286.         for (PackIndex.MutableEntry entry : pack) {
  287.             ObjectId oid = entry.toObjectId();
  288.             if (existing.contains(oid)) {
  289.                 continue;
  290.             }
  291.             existing.add(oid);
  292.             ObjectLoader loader = reader.open(oid);
  293.             inserter.insert(loader.getType(),
  294.                     loader.getSize(),
  295.                     loader.openStream(),
  296.                     true /* create this object even though it's a duplicate */);
  297.         }
  298.     }

  300.     /**
  301.      * Delete old pack files. What is 'old' is defined by specifying a set of
  302.      * old pack files and a set of new pack files. Each pack file contained in
  303.      * old pack files but not contained in new pack files will be deleted. If
  304.      * preserveOldPacks is set, keep a copy of the pack file in the preserve
  305.      * directory. If an expirationDate is set then pack files which are younger
  306.      * than the expirationDate will not be deleted nor preserved.
  307.      * <p>
  308.      * If we're not immediately expiring loose objects, loosen any objects
  309.      * in the old pack files which aren't in the new pack files.
  310.      *
  311.      * @param oldPacks
  312.      * @param newPacks
  313.      * @throws ParseException
  314.      * @throws IOException
  315.      */
  316.     private void deleteOldPacks(Collection<Pack> oldPacks,
  317.             Collection<Pack> newPacks) throws ParseException, IOException {
  318.         HashSet<ObjectId> ids = new HashSet<>();
  319.         for (Pack pack : newPacks) {
  320.             for (PackIndex.MutableEntry entry : pack) {
  321.                 ids.add(entry.toObjectId());
  322.             }
  323.         }
  324.         ObjectReader reader = repo.newObjectReader();
  325.         ObjectDirectory dir = repo.getObjectDatabase();
  326.         ObjectDirectoryInserter inserter = dir.newInserter();
  327.         boolean shouldLoosen = !"now".equals(getPruneExpireStr()) && //$NON-NLS-1$
  328.             getExpireDate() < Long.MAX_VALUE;

  330.         prunePreserved();
  331.         long packExpireDate = getPackExpireDate();
  332.         oldPackLoop: for (Pack oldPack : oldPacks) {
  333.             checkCancelled();
  334.             String oldName = oldPack.getPackName();
  335.             // check whether an old pack file is also among the list of new
  336.             // pack files. Then we must not delete it.
  337.             for (Pack newPack : newPacks)
  338.                 if (oldName.equals(newPack.getPackName()))
  339.                     continue oldPackLoop;

  341.             if (!oldPack.shouldBeKept()
  342.                     && repo.getFS()
  343.                             .lastModifiedInstant(oldPack.getPackFile())
  344.                             .toEpochMilli() < packExpireDate) {
  345.                 if (shouldLoosen) {
  346.                     loosen(inserter, reader, oldPack, ids);
  347.                 }
  348.                 oldPack.close();
  349.                 prunePack(oldPack.getPackFile());
  350.             }
  351.         }

  353.         // close the complete object database. That's my only chance to force
  354.         // rescanning and to detect that certain pack files are now deleted.
  355.         repo.getObjectDatabase().close();
  356.     }

  358.     /**
  359.      * Deletes old pack file, unless 'preserve-oldpacks' is set, in which case it
  360.      * moves the pack file to the preserved directory
  361.      *
  362.      * @param packFile
  363.      * @param deleteOptions
  364.      * @throws IOException
  365.      */
  366.     private void removeOldPack(PackFile packFile, int deleteOptions)
  367.             throws IOException {
  368.         if (pconfig.isPreserveOldPacks()) {
  369.             File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
  370.             FileUtils.mkdir(oldPackDir, true);

  372.             PackFile oldPackFile = packFile
  373.                     .createPreservedForDirectory(oldPackDir);
  374.             FileUtils.rename(packFile, oldPackFile);
  375.         } else {
  376.             FileUtils.delete(packFile, deleteOptions);
  377.         }
  378.     }

  380.     /**
  381.      * Delete the preserved directory including all pack files within
  382.      */
  383.     private void prunePreserved() {
  384.         if (pconfig.isPrunePreserved()) {
  385.             try {
  386.                 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
  387.                         FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
  388.             } catch (IOException e) {
  389.                 // Deletion of the preserved pack files failed. Silently return.
  390.             }
  391.         }
  392.     }

  394.     /**
  395.      * Delete files associated with a single pack file. First try to delete the
  396.      * ".pack" file because on some platforms the ".pack" file may be locked and
  397.      * can't be deleted. In such a case it is better to detect this early and
  398.      * give up on deleting files for this packfile. Otherwise we may delete the
  399.      * ".index" file and when failing to delete the ".pack" file we are left
  400.      * with a ".pack" file without a ".index" file.
  401.      *
  402.      * @param packFile
  403.      */
  404.     private void prunePack(PackFile packFile) {
  405.         try {
  406.             // Delete the .pack file first and if this fails give up on deleting
  407.             // the other files
  408.             int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
  409.             removeOldPack(packFile.create(PackExt.PACK), deleteOptions);

  411.             // The .pack file has been deleted. Delete as many as the other
  412.             // files as you can.
  413.             deleteOptions |= FileUtils.IGNORE_ERRORS;
  414.             for (PackExt ext : PackExt.values()) {
  415.                 if (!PackExt.PACK.equals(ext)) {
  416.                     removeOldPack(packFile.create(ext), deleteOptions);
  417.                 }
  418.             }
  419.         } catch (IOException e) {
  420.             // Deletion of the .pack file failed. Silently return.
  421.         }
  422.     }

  424.     /**
  425.      * Like "git prune-packed" this method tries to prune all loose objects
  426.      * which can be found in packs. If certain objects can't be pruned (e.g.
  427.      * because the filesystem delete operation fails) this is silently ignored.
  428.      *
  429.      * @throws java.io.IOException
  430.      */
  431.     public void prunePacked() throws IOException {
  432.         ObjectDirectory objdb = repo.getObjectDatabase();
  433.         Collection<Pack> packs = objdb.getPacks();
  434.         File objects = repo.getObjectsDirectory();
  435.         String[] fanout = objects.list();

  437.         if (fanout != null && fanout.length > 0) {
  438.             pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
  439.             try {
  440.                 for (String d : fanout) {
  441.                     checkCancelled();
  442.                     pm.update(1);
  443.                     if (d.length() != 2)
  444.                         continue;
  445.                     String[] entries = new File(objects, d).list();
  446.                     if (entries == null)
  447.                         continue;
  448.                     for (String e : entries) {
  449.                         checkCancelled();
  450.                         if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
  451.                             continue;
  452.                         ObjectId id;
  453.                         try {
  454.                             id = ObjectId.fromString(d + e);
  455.                         } catch (IllegalArgumentException notAnObject) {
  456.                             // ignoring the file that does not represent loose
  457.                             // object
  458.                             continue;
  459.                         }
  460.                         boolean found = false;
  461.                         for (Pack p : packs) {
  462.                             checkCancelled();
  463.                             if (p.hasObject(id)) {
  464.                                 found = true;
  465.                                 break;
  466.                             }
  467.                         }
  468.                         if (found)
  469.                             FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
  470.                                     | FileUtils.SKIP_MISSING
  471.                                     | FileUtils.IGNORE_ERRORS);
  472.                     }
  473.                 }
  474.             } finally {
  475.                 pm.endTask();
  476.             }
  477.         }
  478.     }

  480.     /**
  481.      * Like "git prune" this method tries to prune all loose objects which are
  482.      * unreferenced. If certain objects can't be pruned (e.g. because the
  483.      * filesystem delete operation fails) this is silently ignored.
  484.      *
  485.      * @param objectsToKeep
  486.      *            a set of objects which should explicitly not be pruned
  487.      * @throws java.io.IOException
  488.      * @throws java.text.ParseException
  489.      *             If the configuration parameter "gc.pruneexpire" couldn't be
  490.      *             parsed
  491.      */
  492.     public void prune(Set<ObjectId> objectsToKeep) throws IOException,
  493.             ParseException {
  494.         long expireDate = getExpireDate();

  496.         // Collect all loose objects which are old enough, not referenced from
  497.         // the index and not in objectsToKeep
  498.         Map<ObjectId, File> deletionCandidates = new HashMap<>();
  499.         Set<ObjectId> indexObjects = null;
  500.         File objects = repo.getObjectsDirectory();
  501.         String[] fanout = objects.list();
  502.         if (fanout == null || fanout.length == 0) {
  503.             return;
  504.         }
  505.         pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
  506.                 fanout.length);
  507.         try {
  508.             for (String d : fanout) {
  509.                 checkCancelled();
  510.                 pm.update(1);
  511.                 if (d.length() != 2)
  512.                     continue;
  513.                 File dir = new File(objects, d);
  514.                 File[] entries = dir.listFiles();
  515.                 if (entries == null || entries.length == 0) {
  516.                     FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
  517.                     continue;
  518.                 }
  519.                 for (File f : entries) {
  520.                     checkCancelled();
  521.                     String fName = f.getName();
  522.                     if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
  523.                         continue;
  524.                     if (repo.getFS().lastModifiedInstant(f)
  525.                             .toEpochMilli() >= expireDate) {
  526.                         continue;
  527.                     }
  528.                     try {
  529.                         ObjectId id = ObjectId.fromString(d + fName);
  530.                         if (objectsToKeep.contains(id))
  531.                             continue;
  532.                         if (indexObjects == null)
  533.                             indexObjects = listNonHEADIndexObjects();
  534.                         if (indexObjects.contains(id))
  535.                             continue;
  536.                         deletionCandidates.put(id, f);
  537.                     } catch (IllegalArgumentException notAnObject) {
  538.                         // ignoring the file that does not represent loose
  539.                         // object
  540.                     }
  541.                 }
  542.             }
  543.         } finally {
  544.             pm.endTask();
  545.         }

  547.         if (deletionCandidates.isEmpty()) {
  548.             return;
  549.         }

  551.         checkCancelled();

  553.         // From the set of current refs remove all those which have been handled
  554.         // during last repack(). Only those refs will survive which have been
  555.         // added or modified since the last repack. Only these can save existing
  556.         // loose refs from being pruned.
  557.         Collection<Ref> newRefs;
  558.         if (lastPackedRefs == null || lastPackedRefs.isEmpty())
  559.             newRefs = getAllRefs();
  560.         else {
  561.             Map<String, Ref> last = new HashMap<>();
  562.             for (Ref r : lastPackedRefs) {
  563.                 last.put(r.getName(), r);
  564.             }
  565.             newRefs = new ArrayList<>();
  566.             for (Ref r : getAllRefs()) {
  567.                 Ref old = last.get(r.getName());
  568.                 if (!equals(r, old)) {
  569.                     newRefs.add(r);
  570.                 }
  571.             }
  572.         }

  574.         if (!newRefs.isEmpty()) {
  575.             // There are new/modified refs! Check which loose objects are now
  576.             // referenced by these modified refs (or their reflogentries).
  577.             // Remove these loose objects
  578.             // from the deletionCandidates. When the last candidate is removed
  579.             // leave this method.
  580.             ObjectWalk w = new ObjectWalk(repo);
  581.             try {
  582.                 for (Ref cr : newRefs) {
  583.                     checkCancelled();
  584.                     w.markStart(w.parseAny(cr.getObjectId()));
  585.                 }
  586.                 if (lastPackedRefs != null)
  587.                     for (Ref lpr : lastPackedRefs) {
  588.                         w.markUninteresting(w.parseAny(lpr.getObjectId()));
  589.                     }
  590.                 removeReferenced(deletionCandidates, w);
  591.             } finally {
  592.                 w.dispose();
  593.             }
  594.         }

  596.         if (deletionCandidates.isEmpty())
  597.             return;

  599.         // Since we have not left the method yet there are still
  600.         // deletionCandidates. Last chance for these objects not to be pruned is
  601.         // that they are referenced by reflog entries. Even refs which currently
  602.         // point to the same object as during last repack() may have
  603.         // additional reflog entries not handled during last repack()
  604.         ObjectWalk w = new ObjectWalk(repo);
  605.         try {
  606.             for (Ref ar : getAllRefs())
  607.                 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
  608.                     checkCancelled();
  609.                     w.markStart(w.parseAny(id));
  610.                 }
  611.             if (lastPackedRefs != null)
  612.                 for (Ref lpr : lastPackedRefs) {
  613.                     checkCancelled();
  614.                     w.markUninteresting(w.parseAny(lpr.getObjectId()));
  615.                 }
  616.             removeReferenced(deletionCandidates, w);
  617.         } finally {
  618.             w.dispose();
  619.         }

  621.         if (deletionCandidates.isEmpty())
  622.             return;

  624.         checkCancelled();

  626.         // delete all candidates which have survived: these are unreferenced
  627.         // loose objects. Make a last check, though, to avoid deleting objects
  628.         // that could have been referenced while the candidates list was being
  629.         // built (by an incoming push, for example).
  630.         Set<File> touchedFanout = new HashSet<>();
  631.         for (File f : deletionCandidates.values()) {
  632.             if (f.lastModified() < expireDate) {
  633.                 f.delete();
  634.                 touchedFanout.add(f.getParentFile());
  635.             }
  636.         }

  638.         for (File f : touchedFanout) {
  639.             FileUtils.delete(f,
  640.                     FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
  641.         }

  643.         repo.getObjectDatabase().close();
  644.     }

  646.     private long getExpireDate() throws ParseException {
  647.         long expireDate = Long.MAX_VALUE;

  649.         if (expire == null && expireAgeMillis == -1) {
  650.             String pruneExpireStr = getPruneExpireStr();
  651.             if (pruneExpireStr == null)
  652.                 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
  653.             expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
  654.                     .getInstance().getLocale());
  655.             expireAgeMillis = -1;
  656.         }
  657.         if (expire != null)
  658.             expireDate = expire.getTime();
  659.         if (expireAgeMillis != -1)
  660.             expireDate = System.currentTimeMillis() - expireAgeMillis;
  661.         return expireDate;
  662.     }

  664.     private String getPruneExpireStr() {
  665.         return repo.getConfig().getString(
  666.                         ConfigConstants.CONFIG_GC_SECTION, null,
  667.                         ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
  668.     }

  670.     private long getPackExpireDate() throws ParseException {
  671.         long packExpireDate = Long.MAX_VALUE;

  673.         if (packExpire == null && packExpireAgeMillis == -1) {
  674.             String prunePackExpireStr = repo.getConfig().getString(
  675.                     ConfigConstants.CONFIG_GC_SECTION, null,
  676.                     ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
  677.             if (prunePackExpireStr == null)
  678.                 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
  679.             packExpire = GitDateParser.parse(prunePackExpireStr, null,
  680.                     SystemReader.getInstance().getLocale());
  681.             packExpireAgeMillis = -1;
  682.         }
  683.         if (packExpire != null)
  684.             packExpireDate = packExpire.getTime();
  685.         if (packExpireAgeMillis != -1)
  686.             packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
  687.         return packExpireDate;
  688.     }

  690.     /**
  691.      * Remove all entries from a map which key is the id of an object referenced
  692.      * by the given ObjectWalk
  693.      *
  694.      * @param id2File
  695.      * @param w
  696.      * @throws MissingObjectException
  697.      * @throws IncorrectObjectTypeException
  698.      * @throws IOException
  699.      */
  700.     private void removeReferenced(Map<ObjectId, File> id2File,
  701.             ObjectWalk w) throws MissingObjectException,
  702.             IncorrectObjectTypeException, IOException {
  703.         RevObject ro = w.next();
  704.         while (ro != null) {
  705.             checkCancelled();
  706.             if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
  707.                 return;
  708.             }
  709.             ro = w.next();
  710.         }
  711.         ro = w.nextObject();
  712.         while (ro != null) {
  713.             checkCancelled();
  714.             if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
  715.                 return;
  716.             }
  717.             ro = w.nextObject();
  718.         }
  719.     }

  721.     private static boolean equals(Ref r1, Ref r2) {
  722.         if (r1 == null || r2 == null) {
  723.             return false;
  724.         }
  725.         if (r1.isSymbolic()) {
  726.             return r2.isSymbolic() && r1.getTarget().getName()
  727.                     .equals(r2.getTarget().getName());
  728.         }
  729.         return !r2.isSymbolic()
  730.                 && Objects.equals(r1.getObjectId(), r2.getObjectId());
  731.     }

  733.     /**
  734.      * Pack ref storage. For a RefDirectory database, this packs all
  735.      * non-symbolic, loose refs into packed-refs. For Reftable, all of the data
  736.      * is compacted into a single table.
  737.      *
  738.      * @throws java.io.IOException
  739.      */
  740.     public void packRefs() throws IOException {
  741.         RefDatabase refDb = repo.getRefDatabase();
  742.         if (refDb instanceof FileReftableDatabase) {
  743.             // TODO: abstract this more cleanly.
  744.             pm.beginTask(JGitText.get().packRefs, 1);
  745.             try {
  746.                 ((FileReftableDatabase) refDb).compactFully();
  747.             } finally {
  748.                 pm.endTask();
  749.             }
  750.             return;
  751.         }

  753.         Collection<Ref> refs = refDb.getRefsByPrefix(Constants.R_REFS);
  754.         List<String> refsToBePacked = new ArrayList<>(refs.size());
  755.         pm.beginTask(JGitText.get().packRefs, refs.size());
  756.         try {
  757.             for (Ref ref : refs) {
  758.                 checkCancelled();
  759.                 if (!ref.isSymbolic() && ref.getStorage().isLoose())
  760.                     refsToBePacked.add(ref.getName());
  761.                 pm.update(1);
  762.             }
  763.             ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
  764.         } finally {
  765.             pm.endTask();
  766.         }
  767.     }

  769.     /**
  770.      * Packs all objects which reachable from any of the heads into one pack
  771.      * file. Additionally all objects which are not reachable from any head but
  772.      * which are reachable from any of the other refs (e.g. tags), special refs
  773.      * (e.g. FETCH_HEAD) or index are packed into a separate pack file. Objects
  774.      * included in pack files which have a .keep file associated are never
  775.      * repacked. All old pack files which existed before are deleted.
  776.      *
  777.      * @return a collection of the newly created pack files
  778.      * @throws java.io.IOException
  779.      *             when during reading of refs, index, packfiles, objects,
  780.      *             reflog-entries or during writing to the packfiles
  781.      *             {@link java.io.IOException} occurs
  782.      */
  783.     public Collection<Pack> repack() throws IOException {
  784.         Collection<Pack> toBeDeleted = repo.getObjectDatabase().getPacks();

  786.         long time = System.currentTimeMillis();
  787.         Collection<Ref> refsBefore = getAllRefs();

  789.         Set<ObjectId> allHeadsAndTags = new HashSet<>();
  790.         Set<ObjectId> allHeads = new HashSet<>();
  791.         Set<ObjectId> allTags = new HashSet<>();
  792.         Set<ObjectId> nonHeads = new HashSet<>();
  793.         Set<ObjectId> txnHeads = new HashSet<>();
  794.         Set<ObjectId> tagTargets = new HashSet<>();
  795.         Set<ObjectId> indexObjects = listNonHEADIndexObjects();

  797.         for (Ref ref : refsBefore) {
  798.             checkCancelled();
  799.             nonHeads.addAll(listRefLogObjects(ref, 0));
  800.             if (ref.isSymbolic() || ref.getObjectId() == null) {
  801.                 continue;
  802.             }
  803.             if (isHead(ref)) {
  804.                 allHeads.add(ref.getObjectId());
  805.             } else if (isTag(ref)) {
  806.                 allTags.add(ref.getObjectId());
  807.             } else {
  808.                 nonHeads.add(ref.getObjectId());
  809.             }
  810.             if (ref.getPeeledObjectId() != null) {
  811.                 tagTargets.add(ref.getPeeledObjectId());
  812.             }
  813.         }

  815.         List<ObjectIdSet> excluded = new LinkedList<>();
  816.         for (Pack p : repo.getObjectDatabase().getPacks()) {
  817.             checkCancelled();
  818.             if (p.shouldBeKept())
  819.                 excluded.add(p.getIndex());
  820.         }

  822.         // Don't exclude tags that are also branch tips
  823.         allTags.removeAll(allHeads);
  824.         allHeadsAndTags.addAll(allHeads);
  825.         allHeadsAndTags.addAll(allTags);

  827.         // Hoist all branch tips and tags earlier in the pack file
  828.         tagTargets.addAll(allHeadsAndTags);
  829.         nonHeads.addAll(indexObjects);

  831.         // Combine the GC_REST objects into the GC pack if requested
  832.         if (pconfig.getSinglePack()) {
  833.             allHeadsAndTags.addAll(nonHeads);
  834.             nonHeads.clear();
  835.         }

  837.         List<Pack> ret = new ArrayList<>(2);
  838.         Pack heads = null;
  839.         if (!allHeadsAndTags.isEmpty()) {
  840.             heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
  841.                     tagTargets, excluded);
  842.             if (heads != null) {
  843.                 ret.add(heads);
  844.                 excluded.add(0, heads.getIndex());
  845.             }
  846.         }
  847.         if (!nonHeads.isEmpty()) {
  848.             Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
  849.                     tagTargets, excluded);
  850.             if (rest != null)
  851.                 ret.add(rest);
  852.         }
  853.         if (!txnHeads.isEmpty()) {
  854.             Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
  855.                     null, excluded);
  856.             if (txn != null)
  857.                 ret.add(txn);
  858.         }
  859.         try {
  860.             deleteOldPacks(toBeDeleted, ret);
  861.         } catch (ParseException e) {
  862.             // TODO: the exception has to be wrapped into an IOException because
  863.             // throwing the ParseException directly would break the API, instead
  864.             // we should throw a ConfigInvalidException
  865.             throw new IOException(e);
  866.         }
  867.         prunePacked();
  868.         if (repo.getRefDatabase() instanceof RefDirectory) {
  869.             // TODO: abstract this more cleanly.
  870.             deleteEmptyRefsFolders();
  871.         }
  872.         deleteOrphans();
  873.         deleteTempPacksIdx();

  875.         lastPackedRefs = refsBefore;
  876.         lastRepackTime = time;
  877.         return ret;
  878.     }

  880.     private static boolean isHead(Ref ref) {
  881.         return ref.getName().startsWith(Constants.R_HEADS);
  882.     }

  884.     private static boolean isTag(Ref ref) {
  885.         return ref.getName().startsWith(Constants.R_TAGS);
  886.     }

  888.     private void deleteEmptyRefsFolders() throws IOException {
  889.         Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
  890.         // Avoid deleting a folder that was created after the threshold so that concurrent
  891.         // operations trying to create a reference are not impacted
  892.         Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
  893.         try (Stream<Path> entries = Files.list(refs)
  894.                 .filter(Files::isDirectory)) {
  895.             Iterator<Path> iterator = entries.iterator();
  896.             while (iterator.hasNext()) {
  897.                 try (Stream<Path> s = Files.list(iterator.next())) {
  898.                     s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
  899.                 }
  900.             }
  901.         }
  902.     }

  904.     private boolean canBeSafelyDeleted(Path path, Instant threshold) {
  905.         try {
  906.             return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
  907.         }
  908.         catch (IOException e) {
  909.             LOG.warn(MessageFormat.format(
  910.                     JGitText.get().cannotAccessLastModifiedForSafeDeletion,
  911.                     path), e);
  912.             return false;
  913.         }
  914.     }

  916.     private void deleteDir(Path dir) {
  917.         try (Stream<Path> dirs = Files.walk(dir)) {
  918.             dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
  919.                     .forEach(this::delete);
  920.         } catch (IOException e) {
  921.             LOG.error(e.getMessage(), e);
  922.         }
  923.     }

  925.     private boolean isDirectory(Path p) {
  926.         return p.toFile().isDirectory();
  927.     }

  929.     private void delete(Path d) {
  930.         try {
  931.             Files.delete(d);
  932.         } catch (DirectoryNotEmptyException e) {
  933.             // Don't log
  934.         } catch (IOException e) {
  935.             LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
  936.                     e);
  937.         }
  938.     }

  940.     /**
  941.      * Deletes orphans
  942.      * <p>
  943.      * A file is considered an orphan if it is either a "bitmap" or an index
  944.      * file, and its corresponding pack file is missing in the list.
  945.      * </p>
  946.      */
  947.     private void deleteOrphans() {
  948.         Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
  949.         List<String> fileNames = null;
  950.         try (Stream<Path> files = Files.list(packDir)) {
  951.             fileNames = files.map(path -> path.getFileName().toString())
  952.                     .filter(name -> (name.endsWith(PACK_EXT)
  953.                             || name.endsWith(BITMAP_EXT)
  954.                             || name.endsWith(INDEX_EXT)
  955.                             || name.endsWith(KEEP_EXT)))
  956.                     // sort files with same base name in the order:
  957.                     // .pack, .keep, .index, .bitmap to avoid look ahead
  958.                     .sorted(Collections.reverseOrder())
  959.                     .collect(Collectors.toList());
  960.         } catch (IOException e) {
  961.             LOG.error(e.getMessage(), e);
  962.             return;
  963.         }
  964.         if (fileNames == null) {
  965.             return;
  966.         }

  968.         String latestId = null;
  969.         for (String n : fileNames) {
  970.             PackFile pf = new PackFile(packDir.toFile(), n);
  971.             PackExt ext = pf.getPackExt();
  972.             if (ext.equals(PACK) || ext.equals(KEEP)) {
  973.                 latestId = pf.getId();
  974.             }
  975.             if (latestId == null || !pf.getId().equals(latestId)) {
  976.                 // no pack or keep for this id
  977.                 try {
  978.                     FileUtils.delete(pf,
  979.                             FileUtils.RETRY | FileUtils.SKIP_MISSING);
  980.                     LOG.warn(JGitText.get().deletedOrphanInPackDir, pf);
  981.                 } catch (IOException e) {
  982.                     LOG.error(e.getMessage(), e);
  983.                 }
  984.             }
  985.         }
  986.     }

  988.     private void deleteTempPacksIdx() {
  989.         Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
  990.         Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
  991.         if (!Files.exists(packDir)) {
  992.             return;
  993.         }
  994.         try (DirectoryStream<Path> stream =
  995.                 Files.newDirectoryStream(packDir, "gc_*_tmp")) { //$NON-NLS-1$
  996.             stream.forEach(t -> {
  997.                 try {
  998.                     Instant lastModified = Files.getLastModifiedTime(t)
  999.                             .toInstant();
  1000.                     if (lastModified.isBefore(threshold)) {
  1001.                         Files.deleteIfExists(t);
  1002.                     }
  1003.                 } catch (IOException e) {
  1004.                     LOG.error(e.getMessage(), e);
  1005.                 }
  1006.             });
  1007.         } catch (IOException e) {
  1008.             LOG.error(e.getMessage(), e);
  1009.         }
  1010.     }

  1012.     /**
  1013.      * @param ref
  1014.      *            the ref which log should be inspected
  1015.      * @param minTime only reflog entries not older then this time are processed
  1016.      * @return the {@link ObjectId}s contained in the reflog
  1017.      * @throws IOException
  1018.      */
  1019.     private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
  1020.         ReflogReader reflogReader = repo.getReflogReader(ref.getName());
  1021.         if (reflogReader == null) {
  1022.             return Collections.emptySet();
  1023.         }
  1024.         List<ReflogEntry> rlEntries = reflogReader
  1025.                 .getReverseEntries();
  1026.         if (rlEntries == null || rlEntries.isEmpty())
  1027.             return Collections.emptySet();
  1028.         Set<ObjectId> ret = new HashSet<>();
  1029.         for (ReflogEntry e : rlEntries) {
  1030.             if (e.getWho().getWhen().getTime() < minTime)
  1031.                 break;
  1032.             ObjectId newId = e.getNewId();
  1033.             if (newId != null && !ObjectId.zeroId().equals(newId))
  1034.                 ret.add(newId);
  1035.             ObjectId oldId = e.getOldId();
  1036.             if (oldId != null && !ObjectId.zeroId().equals(oldId))
  1037.                 ret.add(oldId);
  1038.         }
  1039.         return ret;
  1040.     }

  1042.     /**
  1043.      * Returns a collection of all refs and additional refs.
  1044.      *
  1045.      * Additional refs which don't start with "refs/" are not returned because
  1046.      * they should not save objects from being garbage collected. Examples for
  1047.      * such references are ORIG_HEAD, MERGE_HEAD, FETCH_HEAD and
  1048.      * CHERRY_PICK_HEAD.
  1049.      *
  1050.      * @return a collection of refs pointing to live objects.
  1051.      * @throws IOException
  1052.      */
  1053.     private Collection<Ref> getAllRefs() throws IOException {
  1054.         RefDatabase refdb = repo.getRefDatabase();
  1055.         Collection<Ref> refs = refdb.getRefs();
  1056.         List<Ref> addl = refdb.getAdditionalRefs();
  1057.         if (!addl.isEmpty()) {
  1058.             List<Ref> all = new ArrayList<>(refs.size() + addl.size());
  1059.             all.addAll(refs);
  1060.             // add additional refs which start with refs/
  1061.             for (Ref r : addl) {
  1062.                 checkCancelled();
  1063.                 if (r.getName().startsWith(Constants.R_REFS)) {
  1064.                     all.add(r);
  1065.                 }
  1066.             }
  1067.             return all;
  1068.         }
  1069.         return refs;
  1070.     }

  1072.     /**
  1073.      * Return a list of those objects in the index which differ from whats in
  1074.      * HEAD
  1075.      *
  1076.      * @return a set of ObjectIds of changed objects in the index
  1077.      * @throws IOException
  1078.      * @throws CorruptObjectException
  1079.      * @throws NoWorkTreeException
  1080.      */
  1081.     private Set<ObjectId> listNonHEADIndexObjects()
  1082.             throws CorruptObjectException, IOException {
  1083.         if (repo.isBare()) {
  1084.             return Collections.emptySet();
  1085.         }
  1086.         try (TreeWalk treeWalk = new TreeWalk(repo)) {
  1087.             treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
  1088.             ObjectId headID = repo.resolve(Constants.HEAD);
  1089.             if (headID != null) {
  1090.                 try (RevWalk revWalk = new RevWalk(repo)) {
  1091.                     treeWalk.addTree(revWalk.parseTree(headID));
  1092.                 }
  1093.             }

  1095.             treeWalk.setFilter(TreeFilter.ANY_DIFF);
  1096.             treeWalk.setRecursive(true);
  1097.             Set<ObjectId> ret = new HashSet<>();

  1099.             while (treeWalk.next()) {
  1100.                 checkCancelled();
  1101.                 ObjectId objectId = treeWalk.getObjectId(0);
  1102.                 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
  1103.                 case FileMode.TYPE_MISSING:
  1104.                 case FileMode.TYPE_GITLINK:
  1105.                     continue;
  1106.                 case FileMode.TYPE_TREE:
  1107.                 case FileMode.TYPE_FILE:
  1108.                 case FileMode.TYPE_SYMLINK:
  1109.                     ret.add(objectId);
  1110.                     continue;
  1111.                 default:
  1112.                     throw new IOException(MessageFormat.format(
  1113.                             JGitText.get().corruptObjectInvalidMode3,
  1114.                             String.format("%o", //$NON-NLS-1$
  1115.                                     Integer.valueOf(treeWalk.getRawMode(0))),
  1116.                             (objectId == null) ? "null" : objectId.name(), //$NON-NLS-1$
  1117.                             treeWalk.getPathString(), //
  1118.                             repo.getIndexFile()));
  1119.                 }
  1120.             }
  1121.             return ret;
  1122.         }
  1123.     }

  1125.     private Pack writePack(@NonNull Set<? extends ObjectId> want,
  1126.             @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
  1127.             Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
  1128.             throws IOException {
  1129.         checkCancelled();
  1130.         File tmpPack = null;
  1131.         Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
  1132.             // INDEX entries must be returned last, so the pack
  1133.             // scanner does pick up the new pack until all the
  1134.             // PackExt entries have been written.
  1135.             if (o1 == o2) {
  1136.                 return 0;
  1137.             }
  1138.             if (o1 == PackExt.INDEX) {
  1139.                 return 1;
  1140.             }
  1141.             if (o2 == PackExt.INDEX) {
  1142.                 return -1;
  1143.             }
  1144.             return Integer.signum(o1.hashCode() - o2.hashCode());
  1145.         });
  1146.         try (PackWriter pw = new PackWriter(
  1147.                 pconfig,
  1148.                 repo.newObjectReader())) {
  1149.             // prepare the PackWriter
  1150.             pw.setDeltaBaseAsOffset(true);
  1151.             pw.setReuseDeltaCommits(false);
  1152.             if (tagTargets != null) {
  1153.                 pw.setTagTargets(tagTargets);
  1154.             }
  1155.             if (excludeObjects != null)
  1156.                 for (ObjectIdSet idx : excludeObjects)
  1157.                     pw.excludeObjects(idx);
  1158.             pw.preparePack(pm, want, have, PackWriter.NONE, tags);
  1159.             if (pw.getObjectCount() == 0)
  1160.                 return null;
  1161.             checkCancelled();

  1163.             // create temporary files
  1164.             ObjectId id = pw.computeName();
  1165.             File packdir = repo.getObjectDatabase().getPackDirectory();
  1166.             packdir.mkdirs();
  1167.             tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$
  1168.             final String tmpBase = tmpPack.getName()
  1169.                     .substring(0, tmpPack.getName().lastIndexOf('.'));
  1170.             File tmpIdx = new File(packdir, tmpBase + ".idx_tmp"); //$NON-NLS-1$
  1171.             tmpExts.put(INDEX, tmpIdx);

  1173.             if (!tmpIdx.createNewFile())
  1174.                 throw new IOException(MessageFormat.format(
  1175.                         JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));

  1177.             // write the packfile
  1178.             try (FileOutputStream fos = new FileOutputStream(tmpPack);
  1179.                     FileChannel channel = fos.getChannel();
  1180.                     OutputStream channelStream = Channels
  1181.                             .newOutputStream(channel)) {
  1182.                 pw.writePack(pm, pm, channelStream);
  1183.                 channel.force(true);
  1184.             }

  1186.             // write the packindex
  1187.             try (FileOutputStream fos = new FileOutputStream(tmpIdx);
  1188.                     FileChannel idxChannel = fos.getChannel();
  1189.                     OutputStream idxStream = Channels
  1190.                             .newOutputStream(idxChannel)) {
  1191.                 pw.writeIndex(idxStream);
  1192.                 idxChannel.force(true);
  1193.             }

  1195.             if (pw.prepareBitmapIndex(pm)) {
  1196.                 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp"); //$NON-NLS-1$
  1197.                 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);

  1199.                 if (!tmpBitmapIdx.createNewFile())
  1200.                     throw new IOException(MessageFormat.format(
  1201.                             JGitText.get().cannotCreateIndexfile,
  1202.                             tmpBitmapIdx.getPath()));

  1204.                 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
  1205.                         FileChannel idxChannel = fos.getChannel();
  1206.                         OutputStream idxStream = Channels
  1207.                                 .newOutputStream(idxChannel)) {
  1208.                     pw.writeBitmapIndex(idxStream);
  1209.                     idxChannel.force(true);
  1210.                 }
  1211.             }

  1213.             // rename the temporary files to real files
  1214.             File packDir = repo.getObjectDatabase().getPackDirectory();
  1215.             PackFile realPack = new PackFile(packDir, id, PackExt.PACK);

  1217.             repo.getObjectDatabase().closeAllPackHandles(realPack);
  1218.             tmpPack.setReadOnly();

  1220.             FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
  1221.             for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
  1222.                 File tmpExt = tmpEntry.getValue();
  1223.                 tmpExt.setReadOnly();

  1225.                 PackFile realExt = new PackFile(packDir, id, tmpEntry.getKey());
  1226.                 try {
  1227.                     FileUtils.rename(tmpExt, realExt,
  1228.                             StandardCopyOption.ATOMIC_MOVE);
  1229.                 } catch (IOException e) {
  1230.                     File newExt = new File(realExt.getParentFile(),
  1231.                             realExt.getName() + ".new"); //$NON-NLS-1$
  1232.                     try {
  1233.                         FileUtils.rename(tmpExt, newExt,
  1234.                                 StandardCopyOption.ATOMIC_MOVE);
  1235.                     } catch (IOException e2) {
  1236.                         newExt = tmpExt;
  1237.                         e = e2;
  1238.                     }
  1239.                     throw new IOException(MessageFormat.format(
  1240.                             JGitText.get().panicCantRenameIndexFile, newExt,
  1241.                             realExt), e);
  1242.                 }
  1243.             }
  1244.             boolean interrupted = false;
  1245.             try {
  1246.                 FileSnapshot snapshot = FileSnapshot.save(realPack);
  1247.                 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
  1248.                     snapshot.waitUntilNotRacy();
  1249.                 }
  1250.             } catch (InterruptedException e) {
  1251.                 interrupted = true;
  1252.             }
  1253.             try {
  1254.                 return repo.getObjectDatabase().openPack(realPack);
  1255.             } finally {
  1256.                 if (interrupted) {
  1257.                     // Re-set interrupted flag
  1258.                     Thread.currentThread().interrupt();
  1259.                 }
  1260.             }
  1261.         } finally {
  1262.             if (tmpPack != null && tmpPack.exists())
  1263.                 tmpPack.delete();
  1264.             for (File tmpExt : tmpExts.values()) {
  1265.                 if (tmpExt.exists())
  1266.                     tmpExt.delete();
  1267.             }
  1268.         }
  1269.     }

  1271.     private void checkCancelled() throws CancelledException {
  1272.         if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
  1273.             throw new CancelledException(JGitText.get().operationCanceled);
  1274.         }
  1275.     }

  1277.     /**
  1278.      * A class holding statistical data for a FileRepository regarding how many
  1279.      * objects are stored as loose or packed objects
  1280.      */
  1281.     public static class RepoStatistics {
  1282.         /**
  1283.          * The number of objects stored in pack files. If the same object is
  1284.          * stored in multiple pack files then it is counted as often as it
  1285.          * occurs in pack files.
  1286.          */
  1287.         public long numberOfPackedObjects;

  1289.         /**
  1290.          * The number of pack files
  1291.          */
  1292.         public long numberOfPackFiles;

  1294.         /**
  1295.          * The number of objects stored as loose objects.
  1296.          */
  1297.         public long numberOfLooseObjects;

  1299.         /**
  1300.          * The sum of the sizes of all files used to persist loose objects.
  1301.          */
  1302.         public long sizeOfLooseObjects;

  1304.         /**
  1305.          * The sum of the sizes of all pack files.
  1306.          */
  1307.         public long sizeOfPackedObjects;

  1309.         /**
  1310.          * The number of loose refs.
  1311.          */
  1312.         public long numberOfLooseRefs;

  1314.         /**
  1315.          * The number of refs stored in pack files.
  1316.          */
  1317.         public long numberOfPackedRefs;

  1319.         /**
  1320.          * The number of bitmaps in the bitmap indices.
  1321.          */
  1322.         public long numberOfBitmaps;

  1324.         @Override
  1325.         public String toString() {
  1326.             final StringBuilder b = new StringBuilder();
  1327.             b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$
  1328.             b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$
  1329.             b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$
  1330.             b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$
  1331.             b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$
  1332.             b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
  1333.             b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
  1334.             b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
  1335.             return b.toString();
  1336.         }
  1337.     }

  1339.     /**
  1340.      * Returns information about objects and pack files for a FileRepository.
  1341.      *
  1342.      * @return information about objects and pack files for a FileRepository
  1343.      * @throws java.io.IOException
  1344.      */
  1345.     public RepoStatistics getStatistics() throws IOException {
  1346.         RepoStatistics ret = new RepoStatistics();
  1347.         Collection<Pack> packs = repo.getObjectDatabase().getPacks();
  1348.         for (Pack p : packs) {
  1349.             ret.numberOfPackedObjects += p.getIndex().getObjectCount();
  1350.             ret.numberOfPackFiles++;
  1351.             ret.sizeOfPackedObjects += p.getPackFile().length();
  1352.             if (p.getBitmapIndex() != null)
  1353.                 ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount();
  1354.         }
  1355.         File objDir = repo.getObjectsDirectory();
  1356.         String[] fanout = objDir.list();
  1357.         if (fanout != null && fanout.length > 0) {
  1358.             for (String d : fanout) {
  1359.                 if (d.length() != 2)
  1360.                     continue;
  1361.                 File[] entries = new File(objDir, d).listFiles();
  1362.                 if (entries == null)
  1363.                     continue;
  1364.                 for (File f : entries) {
  1365.                     if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
  1366.                         continue;
  1367.                     ret.numberOfLooseObjects++;
  1368.                     ret.sizeOfLooseObjects += f.length();
  1369.                 }
  1370.             }
  1371.         }

  1373.         RefDatabase refDb = repo.getRefDatabase();
  1374.         for (Ref r : refDb.getRefs()) {
  1375.             Storage storage = r.getStorage();
  1376.             if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
  1377.                 ret.numberOfLooseRefs++;
  1378.             if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
  1379.                 ret.numberOfPackedRefs++;
  1380.         }

  1382.         return ret;
  1383.     }

  1385.     /**
  1386.      * Set the progress monitor used for garbage collection methods.
  1387.      *
  1388.      * @param pm a {@link org.eclipse.jgit.lib.ProgressMonitor} object.
  1389.      * @return this
  1390.      */
  1391.     public GC setProgressMonitor(ProgressMonitor pm) {
  1392.         this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
  1393.         return this;
  1394.     }

  1396.     /**
  1397.      * During gc() or prune() each unreferenced, loose object which has been
  1398.      * created or modified in the last <code>expireAgeMillis</code> milliseconds
  1399.      * will not be pruned. Only older objects may be pruned. If set to 0 then
  1400.      * every object is a candidate for pruning.
  1401.      *
  1402.      * @param expireAgeMillis
  1403.      *            minimal age of objects to be pruned in milliseconds.
  1404.      */
  1405.     public void setExpireAgeMillis(long expireAgeMillis) {
  1406.         this.expireAgeMillis = expireAgeMillis;
  1407.         expire = null;
  1408.     }

  1410.     /**
  1411.      * During gc() or prune() packfiles which are created or modified in the
  1412.      * last <code>packExpireAgeMillis</code> milliseconds will not be deleted.
  1413.      * Only older packfiles may be deleted. If set to 0 then every packfile is a
  1414.      * candidate for deletion.
  1415.      *
  1416.      * @param packExpireAgeMillis
  1417.      *            minimal age of packfiles to be deleted in milliseconds.
  1418.      */
  1419.     public void setPackExpireAgeMillis(long packExpireAgeMillis) {
  1420.         this.packExpireAgeMillis = packExpireAgeMillis;
  1421.         expire = null;
  1422.     }

  1424.     /**
  1425.      * Set the PackConfig used when (re-)writing packfiles. This allows to
  1426.      * influence how packs are written and to implement something similar to
  1427.      * "git gc --aggressive"
  1428.      *
  1429.      * @param pconfig
  1430.      *            the {@link org.eclipse.jgit.storage.pack.PackConfig} used when
  1431.      *            writing packs
  1432.      */
  1433.     public void setPackConfig(@NonNull PackConfig pconfig) {
  1434.         this.pconfig = pconfig;
  1435.     }

  1437.     /**
  1438.      * During gc() or prune() each unreferenced, loose object which has been
  1439.      * created or modified after or at <code>expire</code> will not be pruned.
  1440.      * Only older objects may be pruned. If set to null then every object is a
  1441.      * candidate for pruning.
  1442.      *
  1443.      * @param expire
  1444.      *            instant in time which defines object expiration
  1445.      *            objects with modification time before this instant are expired
  1446.      *            objects with modification time newer or equal to this instant
  1447.      *            are not expired
  1448.      */
  1449.     public void setExpire(Date expire) {
  1450.         this.expire = expire;
  1451.         expireAgeMillis = -1;
  1452.     }

  1454.     /**
  1455.      * During gc() or prune() packfiles which are created or modified after or
  1456.      * at <code>packExpire</code> will not be deleted. Only older packfiles may
  1457.      * be deleted. If set to null then every packfile is a candidate for
  1458.      * deletion.
  1459.      *
  1460.      * @param packExpire
  1461.      *            instant in time which defines packfile expiration
  1462.      */
  1463.     public void setPackExpire(Date packExpire) {
  1464.         this.packExpire = packExpire;
  1465.         packExpireAgeMillis = -1;
  1466.     }

  1468.     /**
  1469.      * Set the {@code gc --auto} option.
  1470.      *
  1471.      * With this option, gc checks whether any housekeeping is required; if not,
  1472.      * it exits without performing any work. Some JGit commands run
  1473.      * {@code gc --auto} after performing operations that could create many
  1474.      * loose objects.
  1475.      * <p>
  1476.      * Housekeeping is required if there are too many loose objects or too many
  1477.      * packs in the repository. If the number of loose objects exceeds the value
  1478.      * of the gc.auto option JGit GC consolidates all existing packs into a
  1479.      * single pack (equivalent to {@code -A} option), whereas git-core would
  1480.      * combine all loose objects into a single pack using {@code repack -d -l}.
  1481.      * Setting the value of {@code gc.auto} to 0 disables automatic packing of
  1482.      * loose objects.
  1483.      * <p>
  1484.      * If the number of packs exceeds the value of {@code gc.autoPackLimit},
  1485.      * then existing packs (except those marked with a .keep file) are
  1486.      * consolidated into a single pack by using the {@code -A} option of repack.
  1487.      * Setting {@code gc.autoPackLimit} to 0 disables automatic consolidation of
  1488.      * packs.
  1489.      * <p>
  1490.      * Like git the following jgit commands run auto gc:
  1491.      * <ul>
  1492.      * <li>fetch</li>
  1493.      * <li>merge</li>
  1494.      * <li>rebase</li>
  1495.      * <li>receive-pack</li>
  1496.      * </ul>
  1497.      * The auto gc for receive-pack can be suppressed by setting the config
  1498.      * option {@code receive.autogc = false}
  1499.      *
  1500.      * @param auto
  1501.      *            defines whether gc should do automatic housekeeping
  1502.      */
  1503.     public void setAuto(boolean auto) {
  1504.         this.automatic = auto;
  1505.     }

  1507.     /**
  1508.      * @param background
  1509.      *            whether to run the gc in a background thread.
  1510.      */
  1511.     void setBackground(boolean background) {
  1512.         this.background = background;
  1513.     }

  1515.     private boolean needGc() {
  1516.         if (tooManyPacks()) {
  1517.             addRepackAllOption();
  1518.         } else {
  1519.             return tooManyLooseObjects();
  1520.         }
  1521.         // TODO run pre-auto-gc hook, if it fails return false
  1522.         return true;
  1523.     }

  1525.     private void addRepackAllOption() {
  1526.         // TODO: if JGit GC is enhanced to support repack's option -l this
  1527.         // method needs to be implemented
  1528.     }

  1530.     /**
  1531.      * @return {@code true} if number of packs &gt; gc.autopacklimit (default
  1532.      *         50)
  1533.      */
  1534.     boolean tooManyPacks() {
  1535.         int autopacklimit = repo.getConfig().getInt(
  1536.                 ConfigConstants.CONFIG_GC_SECTION,
  1537.                 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
  1538.                 DEFAULT_AUTOPACKLIMIT);
  1539.         if (autopacklimit <= 0) {
  1540.             return false;
  1541.         }
  1542.         // JGit always creates two packfiles, one for the objects reachable from
  1543.         // branches, and another one for the rest
  1544.         return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
  1545.     }

  1547.     /**
  1548.      * Quickly estimate number of loose objects, SHA1 is distributed evenly so
  1549.      * counting objects in one directory (bucket 17) is sufficient
  1550.      *
  1551.      * @return {@code true} if number of loose objects &gt; gc.auto (default
  1552.      *         6700)
  1553.      */
  1554.     boolean tooManyLooseObjects() {
  1555.         int auto = getLooseObjectLimit();
  1556.         if (auto <= 0) {
  1557.             return false;
  1558.         }
  1559.         int n = 0;
  1560.         int threshold = (auto + 255) / 256;
  1561.         Path dir = repo.getObjectsDirectory().toPath().resolve("17"); //$NON-NLS-1$
  1562.         if (!dir.toFile().exists()) {
  1563.             return false;
  1564.         }
  1565.         try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
  1566.                     Path fileName = file.getFileName();
  1567.                     return file.toFile().isFile() && fileName != null
  1568.                             && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
  1569.                                     .matches();
  1570.                 })) {
  1571.             for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
  1572.                     .next()) {
  1573.                 if (++n > threshold) {
  1574.                     return true;
  1575.                 }
  1576.             }
  1577.         } catch (IOException e) {
  1578.             LOG.error(e.getMessage(), e);
  1579.         }
  1580.         return false;
  1581.     }

  1583.     private int getLooseObjectLimit() {
  1584.         return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
  1585.                 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
  1586.     }
  1587. }
Created with JaCoCo 0.8.7.202105040129