PackDirectory.java

  1. /*
  2.  * Copyright (C) 2009, Google Inc. and others
  3.  *
  4.  * This program and the accompanying materials are made available under the
  5.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  6.  * https://www.eclipse.org/org/documents/edl-v10.php.
  7.  *
  8.  * SPDX-License-Identifier: BSD-3-Clause
  9.  */

  11. package org.eclipse.jgit.internal.storage.file;

  13. import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
  14. import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
  15. import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;

  17. import java.io.File;
  18. import java.io.FileNotFoundException;
  19. import java.io.IOException;
  20. import java.text.MessageFormat;
  21. import java.util.ArrayList;
  22. import java.util.Arrays;
  23. import java.util.Collection;
  24. import java.util.Collections;
  25. import java.util.EnumMap;
  26. import java.util.HashMap;
  27. import java.util.List;
  28. import java.util.Map;
  29. import java.util.Set;
  30. import java.util.concurrent.atomic.AtomicReference;

  32. import org.eclipse.jgit.annotations.Nullable;
  33. import org.eclipse.jgit.errors.CorruptObjectException;
  34. import org.eclipse.jgit.errors.PackInvalidException;
  35. import org.eclipse.jgit.errors.PackMismatchException;
  36. import org.eclipse.jgit.errors.SearchForReuseTimeout;
  37. import org.eclipse.jgit.internal.JGitText;
  38. import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
  39. import org.eclipse.jgit.internal.storage.pack.PackExt;
  40. import org.eclipse.jgit.internal.storage.pack.PackWriter;
  41. import org.eclipse.jgit.lib.AbbreviatedObjectId;
  42. import org.eclipse.jgit.lib.AnyObjectId;
  43. import org.eclipse.jgit.lib.Config;
  44. import org.eclipse.jgit.lib.ConfigConstants;
  45. import org.eclipse.jgit.lib.ObjectId;
  46. import org.eclipse.jgit.lib.ObjectLoader;
  47. import org.eclipse.jgit.util.FileUtils;
  48. import org.slf4j.Logger;
  49. import org.slf4j.LoggerFactory;

  51. /**
  52.  * Traditional file system packed objects directory handler.
  53.  * <p>
  54.  * This is the {@link org.eclipse.jgit.internal.storage.file.Pack}s object
  55.  * representation for a Git object database, where objects are stored in
  56.  * compressed containers known as
  57.  * {@link org.eclipse.jgit.internal.storage.file.Pack}s.
  58.  */
  59. class PackDirectory {
  60.     private final static Logger LOG = LoggerFactory
  61.             .getLogger(PackDirectory.class);

  63.     private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY,
  64.             new Pack[0]);

  66.     private final Config config;

  68.     private final File directory;

  70.     private final AtomicReference<PackList> packList;

  72.     /**
  73.      * Initialize a reference to an on-disk 'pack' directory.
  74.      *
  75.      * @param config
  76.      *            configuration this directory consults for write settings.
  77.      * @param directory
  78.      *            the location of the {@code pack} directory.
  79.      */
  80.     PackDirectory(Config config, File directory) {
  81.         this.config = config;
  82.         this.directory = directory;
  83.         packList = new AtomicReference<>(NO_PACKS);
  84.     }

  86.     /**
  87.      * Getter for the field {@code directory}.
  88.      *
  89.      * @return the location of the {@code pack} directory.
  90.      */
  91.     File getDirectory() {
  92.         return directory;
  93.     }

  95.     void create() throws IOException {
  96.         FileUtils.mkdir(directory);
  97.     }

  99.     void close() {
  100.         PackList packs = packList.get();
  101.         if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) {
  102.             for (Pack p : packs.packs) {
  103.                 p.close();
  104.             }
  105.         }
  106.     }

  108.     Collection<Pack> getPacks() {
  109.         PackList list = packList.get();
  110.         if (list == NO_PACKS) {
  111.             list = scanPacks(list);
  112.         }
  113.         Pack[] packs = list.packs;
  114.         return Collections.unmodifiableCollection(Arrays.asList(packs));
  115.     }

  117.     /** {@inheritDoc} */
  118.     @Override
  119.     public String toString() {
  120.         return "PackDirectory[" + getDirectory() + "]"; //$NON-NLS-1$ //$NON-NLS-2$
  121.     }

  123.     /**
  124.      * Does the requested object exist in this PackDirectory?
  125.      *
  126.      * @param objectId
  127.      *            identity of the object to test for existence of.
  128.      * @return {@code true} if the specified object is stored in this PackDirectory.
  129.      */
  130.     boolean has(AnyObjectId objectId) {
  131.         return getPack(objectId) != null;
  132.     }

  134.     /**
  135.      * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} for the
  136.      * specified object if it is stored in this PackDirectory.
  137.      *
  138.      * @param objectId
  139.      *            identity of the object to find the Pack for.
  140.      * @return {@link org.eclipse.jgit.internal.storage.file.Pack} which
  141.      *         contains the specified object or {@code null} if it is not stored
  142.      *         in this PackDirectory.
  143.      */
  144.     @Nullable
  145.     Pack getPack(AnyObjectId objectId) {
  146.         PackList pList;
  147.         do {
  148.             pList = packList.get();
  149.             for (Pack p : pList.packs) {
  150.                 try {
  151.                     if (p.hasObject(objectId)) {
  152.                         return p;
  153.                     }
  154.                 } catch (IOException e) {
  155.                     // The hasObject call should have only touched the index, so
  156.                     // any failure here indicates the index is unreadable by
  157.                     // this process, and the pack is likewise not readable.
  158.                     LOG.warn(MessageFormat.format(
  159.                             JGitText.get().unableToReadPackfile,
  160.                             p.getPackFile().getAbsolutePath()), e);
  161.                     remove(p);
  162.                 }
  163.             }
  164.         } while (searchPacksAgain(pList));
  165.         return null;
  166.     }

  168.     /**
  169.      * Find objects matching the prefix abbreviation.
  170.      *
  171.      * @param matches
  172.      *            set to add any located ObjectIds to. This is an output
  173.      *            parameter.
  174.      * @param id
  175.      *            prefix to search for.
  176.      * @param matchLimit
  177.      *            maximum number of results to return. At most this many
  178.      *            ObjectIds should be added to matches before returning.
  179.      * @return {@code true} if the matches were exhausted before reaching
  180.      *         {@code maxLimit}.
  181.      */
  182.     boolean resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
  183.             int matchLimit) {
  184.         // Go through the packs once. If we didn't find any resolutions
  185.         // scan for new packs and check once more.
  186.         int oldSize = matches.size();
  187.         PackList pList;
  188.         do {
  189.             pList = packList.get();
  190.             for (Pack p : pList.packs) {
  191.                 try {
  192.                     p.resolve(matches, id, matchLimit);
  193.                     p.resetTransientErrorCount();
  194.                 } catch (IOException e) {
  195.                     handlePackError(e, p);
  196.                 }
  197.                 if (matches.size() > matchLimit) {
  198.                     return false;
  199.                 }
  200.             }
  201.         } while (matches.size() == oldSize && searchPacksAgain(pList));
  202.         return true;
  203.     }

  205.     ObjectLoader open(WindowCursor curs, AnyObjectId objectId) {
  206.         PackList pList;
  207.         do {
  208.             SEARCH: for (;;) {
  209.                 pList = packList.get();
  210.                 for (Pack p : pList.packs) {
  211.                     try {
  212.                         ObjectLoader ldr = p.get(curs, objectId);
  213.                         p.resetTransientErrorCount();
  214.                         if (ldr != null)
  215.                             return ldr;
  216.                     } catch (PackMismatchException e) {
  217.                         // Pack was modified; refresh the entire pack list.
  218.                         if (searchPacksAgain(pList)) {
  219.                             continue SEARCH;
  220.                         }
  221.                     } catch (IOException e) {
  222.                         handlePackError(e, p);
  223.                     }
  224.                 }
  225.                 break SEARCH;
  226.             }
  227.         } while (searchPacksAgain(pList));
  228.         return null;
  229.     }

  231.     long getSize(WindowCursor curs, AnyObjectId id) {
  232.         PackList pList;
  233.         do {
  234.             SEARCH: for (;;) {
  235.                 pList = packList.get();
  236.                 for (Pack p : pList.packs) {
  237.                     try {
  238.                         long len = p.getObjectSize(curs, id);
  239.                         p.resetTransientErrorCount();
  240.                         if (0 <= len) {
  241.                             return len;
  242.                         }
  243.                     } catch (PackMismatchException e) {
  244.                         // Pack was modified; refresh the entire pack list.
  245.                         if (searchPacksAgain(pList)) {
  246.                             continue SEARCH;
  247.                         }
  248.                     } catch (IOException e) {
  249.                         handlePackError(e, p);
  250.                     }
  251.                 }
  252.                 break SEARCH;
  253.             }
  254.         } while (searchPacksAgain(pList));
  255.         return -1;
  256.     }

  258.     void selectRepresentation(PackWriter packer, ObjectToPack otp,
  259.             WindowCursor curs) {
  260.         PackList pList = packList.get();
  261.         SEARCH: for (;;) {
  262.             for (Pack p : pList.packs) {
  263.                 try {
  264.                     LocalObjectRepresentation rep = p.representation(curs, otp);
  265.                     p.resetTransientErrorCount();
  266.                     if (rep != null) {
  267.                         packer.select(otp, rep);
  268.                         packer.checkSearchForReuseTimeout();
  269.                     }
  270.                 } catch (SearchForReuseTimeout e) {
  271.                     break SEARCH;
  272.                 } catch (PackMismatchException e) {
  273.                     // Pack was modified; refresh the entire pack list.
  274.                     //
  275.                     pList = scanPacks(pList);
  276.                     continue SEARCH;
  277.                 } catch (IOException e) {
  278.                     handlePackError(e, p);
  279.                 }
  280.             }
  281.             break SEARCH;
  282.         }
  283.     }

  285.     private void handlePackError(IOException e, Pack p) {
  286.         String warnTmpl = null;
  287.         int transientErrorCount = 0;
  288.         String errTmpl = JGitText.get().exceptionWhileReadingPack;
  289.         if ((e instanceof CorruptObjectException)
  290.                 || (e instanceof PackInvalidException)) {
  291.             warnTmpl = JGitText.get().corruptPack;
  292.             LOG.warn(MessageFormat.format(warnTmpl,
  293.                     p.getPackFile().getAbsolutePath()), e);
  294.             // Assume the pack is corrupted, and remove it from the list.
  295.             remove(p);
  296.         } else if (e instanceof FileNotFoundException) {
  297.             if (p.getPackFile().exists()) {
  298.                 errTmpl = JGitText.get().packInaccessible;
  299.                 transientErrorCount = p.incrementTransientErrorCount();
  300.             } else {
  301.                 warnTmpl = JGitText.get().packWasDeleted;
  302.                 remove(p);
  303.             }
  304.         } else if (FileUtils.isStaleFileHandleInCausalChain(e)) {
  305.             warnTmpl = JGitText.get().packHandleIsStale;
  306.             remove(p);
  307.         } else {
  308.             transientErrorCount = p.incrementTransientErrorCount();
  309.         }
  310.         if (warnTmpl != null) {
  311.             LOG.warn(MessageFormat.format(warnTmpl,
  312.                     p.getPackFile().getAbsolutePath()), e);
  313.         } else {
  314.             if (doLogExponentialBackoff(transientErrorCount)) {
  315.                 // Don't remove the pack from the list, as the error may be
  316.                 // transient.
  317.                 LOG.error(MessageFormat.format(errTmpl,
  318.                         p.getPackFile().getAbsolutePath(),
  319.                         Integer.valueOf(transientErrorCount)), e);
  320.             }
  321.         }
  322.     }

  324.     /**
  325.      * @param n
  326.      *            count of consecutive failures
  327.      * @return @{code true} if i is a power of 2
  328.      */
  329.     private boolean doLogExponentialBackoff(int n) {
  330.         return (n & (n - 1)) == 0;
  331.     }

  333.     boolean searchPacksAgain(PackList old) {
  334.         // Whether to trust the pack folder's modification time. If set
  335.         // to false we will always scan the .git/objects/pack folder to
  336.         // check for new pack files. If set to true (default) we use the
  337.         // lastmodified attribute of the folder and assume that no new
  338.         // pack files can be in this folder if his modification time has
  339.         // not changed.
  340.         boolean trustFolderStat = config.getBoolean(
  341.                 ConfigConstants.CONFIG_CORE_SECTION,
  342.                 ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true);

  344.         return ((!trustFolderStat) || old.snapshot.isModified(directory))
  345.                 && old != scanPacks(old);
  346.     }

  348.     void insert(Pack pack) {
  349.         PackList o, n;
  350.         do {
  351.             o = packList.get();

  353.             // If the pack in question is already present in the list
  354.             // (picked up by a concurrent thread that did a scan?) we
  355.             // do not want to insert it a second time.
  356.             //
  357.             final Pack[] oldList = o.packs;
  358.             final String name = pack.getPackFile().getName();
  359.             for (Pack p : oldList) {
  360.                 if (name.equals(p.getPackFile().getName())) {
  361.                     return;
  362.                 }
  363.             }

  365.             final Pack[] newList = new Pack[1 + oldList.length];
  366.             newList[0] = pack;
  367.             System.arraycopy(oldList, 0, newList, 1, oldList.length);
  368.             n = new PackList(o.snapshot, newList);
  369.         } while (!packList.compareAndSet(o, n));
  370.     }

  372.     private void remove(Pack deadPack) {
  373.         PackList o, n;
  374.         do {
  375.             o = packList.get();

  377.             final Pack[] oldList = o.packs;
  378.             final int j = indexOf(oldList, deadPack);
  379.             if (j < 0) {
  380.                 break;
  381.             }

  383.             final Pack[] newList = new Pack[oldList.length - 1];
  384.             System.arraycopy(oldList, 0, newList, 0, j);
  385.             System.arraycopy(oldList, j + 1, newList, j, newList.length - j);
  386.             n = new PackList(o.snapshot, newList);
  387.         } while (!packList.compareAndSet(o, n));
  388.         deadPack.close();
  389.     }

  391.     private static int indexOf(Pack[] list, Pack pack) {
  392.         for (int i = 0; i < list.length; i++) {
  393.             if (list[i] == pack) {
  394.                 return i;
  395.             }
  396.         }
  397.         return -1;
  398.     }

  400.     private PackList scanPacks(PackList original) {
  401.         synchronized (packList) {
  402.             PackList o, n;
  403.             do {
  404.                 o = packList.get();
  405.                 if (o != original) {
  406.                     // Another thread did the scan for us, while we
  407.                     // were blocked on the monitor above.
  408.                     //
  409.                     return o;
  410.                 }
  411.                 n = scanPacksImpl(o);
  412.                 if (n == o) {
  413.                     return n;
  414.                 }
  415.             } while (!packList.compareAndSet(o, n));
  416.             return n;
  417.         }
  418.     }

  420.     private PackList scanPacksImpl(PackList old) {
  421.         final Map<String, Pack> forReuse = reuseMap(old);
  422.         final FileSnapshot snapshot = FileSnapshot.save(directory);
  423.         Map<String, Map<PackExt, PackFile>> packFilesByExtById = getPackFilesByExtById();
  424.         List<Pack> list = new ArrayList<>(packFilesByExtById.size());
  425.         boolean foundNew = false;
  426.         for (Map<PackExt, PackFile> packFilesByExt : packFilesByExtById
  427.                 .values()) {
  428.             PackFile packFile = packFilesByExt.get(PACK);
  429.             if (packFile == null || !packFilesByExt.containsKey(INDEX)) {
  430.                 // Sometimes C Git's HTTP fetch transport leaves a
  431.                 // .idx file behind and does not download the .pack.
  432.                 // We have to skip over such useless indexes.
  433.                 // Also skip if we don't have any index for this id
  434.                 continue;
  435.             }

  437.             Pack oldPack = forReuse.get(packFile.getName());
  438.             if (oldPack != null
  439.                     && !oldPack.getFileSnapshot().isModified(packFile)) {
  440.                 forReuse.remove(packFile.getName());
  441.                 list.add(oldPack);
  442.                 continue;
  443.             }

  445.             list.add(new Pack(packFile, packFilesByExt.get(BITMAP_INDEX)));
  446.             foundNew = true;
  447.         }

  449.         // If we did not discover any new files, the modification time was not
  450.         // changed, and we did not remove any files, then the set of files is
  451.         // the same as the set we were given. Instead of building a new object
  452.         // return the same collection.
  453.         //
  454.         if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) {
  455.             old.snapshot.setClean(snapshot);
  456.             return old;
  457.         }

  459.         for (Pack p : forReuse.values()) {
  460.             p.close();
  461.         }

  463.         if (list.isEmpty()) {
  464.             return new PackList(snapshot, NO_PACKS.packs);
  465.         }

  467.         final Pack[] r = list.toArray(new Pack[0]);
  468.         Arrays.sort(r, Pack.SORT);
  469.         return new PackList(snapshot, r);
  470.     }

  472.     private static Map<String, Pack> reuseMap(PackList old) {
  473.         final Map<String, Pack> forReuse = new HashMap<>();
  474.         for (Pack p : old.packs) {
  475.             if (p.invalid()) {
  476.                 // The pack instance is corrupted, and cannot be safely used
  477.                 // again. Do not include it in our reuse map.
  478.                 //
  479.                 p.close();
  480.                 continue;
  481.             }

  483.             final Pack prior = forReuse.put(p.getPackFile().getName(), p);
  484.             if (prior != null) {
  485.                 // This should never occur. It should be impossible for us
  486.                 // to have two pack files with the same name, as all of them
  487.                 // came out of the same directory. If it does, we promised to
  488.                 // close any PackFiles we did not reuse, so close the second,
  489.                 // readers are likely to be actively using the first.
  490.                 //
  491.                 forReuse.put(prior.getPackFile().getName(), prior);
  492.                 p.close();
  493.             }
  494.         }
  495.         return forReuse;
  496.     }

  498.     /**
  499.      * Scans the pack directory for
  500.      * {@link org.eclipse.jgit.internal.storage.file.PackFile}s and returns them
  501.      * organized by their extensions and their pack ids
  502.      *
  503.      * Skips files in the directory that we cannot create a
  504.      * {@link org.eclipse.jgit.internal.storage.file.PackFile} for.
  505.      *
  506.      * @return a map of {@link org.eclipse.jgit.internal.storage.file.PackFile}s
  507.      *         and {@link org.eclipse.jgit.internal.storage.pack.PackExt}s keyed
  508.      *         by pack ids
  509.      */
  510.     private Map<String, Map<PackExt, PackFile>> getPackFilesByExtById() {
  511.         final String[] nameList = directory.list();
  512.         if (nameList == null) {
  513.             return Collections.emptyMap();
  514.         }
  515.         Map<String, Map<PackExt, PackFile>> packFilesByExtById = new HashMap<>(
  516.                 nameList.length / 2); // assume roughly 2 files per id
  517.         for (String name : nameList) {
  518.             try {
  519.                 PackFile pack = new PackFile(directory, name);
  520.                 if (pack.getPackExt() != null) {
  521.                     Map<PackExt, PackFile> packByExt = packFilesByExtById
  522.                             .get(pack.getId());
  523.                     if (packByExt == null) {
  524.                         packByExt = new EnumMap<>(PackExt.class);
  525.                         packFilesByExtById.put(pack.getId(), packByExt);
  526.                     }
  527.                     packByExt.put(pack.getPackExt(), pack);
  528.                 }
  529.             } catch (IllegalArgumentException e) {
  530.                 continue;
  531.             }
  532.         }
  533.         return packFilesByExtById;
  534.     }

  536.     static final class PackList {
  537.         /** State just before reading the pack directory. */
  538.         final FileSnapshot snapshot;

  540.         /** All known packs, sorted by {@link Pack#SORT}. */
  541.         final Pack[] packs;

  543.         PackList(FileSnapshot monitor, Pack[] packs) {
  544.             this.snapshot = monitor;
  545.             this.packs = packs;
  546.         }
  547.     }
  548. }
Created with JaCoCo 0.8.7.202105040129