package filesystem import ( "io" "os" "time" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/objfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" ) type ObjectStorage struct { options Options // objectCache is an object cache uses to cache delta's bases and also recently // loaded loose objects objectCache cache.Object dir *dotgit.DotGit index map[plumbing.Hash]idxfile.Index } // NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. func NewObjectStorage(dir *dotgit.DotGit, objectCache cache.Object) *ObjectStorage { return NewObjectStorageWithOptions(dir, objectCache, Options{}) } // NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options func NewObjectStorageWithOptions(dir *dotgit.DotGit, objectCache cache.Object, ops Options) *ObjectStorage { return &ObjectStorage{ options: ops, objectCache: objectCache, dir: dir, } } func (s *ObjectStorage) requireIndex() error { if s.index != nil { return nil } s.index = make(map[plumbing.Hash]idxfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err } for _, h := range packs { if err := s.loadIdxFile(h); err != nil { return err } } return nil } // Reindex indexes again all packfiles. Useful if git changed packfiles externally func (s *ObjectStorage) Reindex() { s.index = nil } func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { f, err := s.dir.ObjectPackIdx(h) if err != nil { return err } defer ioutil.CheckClose(f, &err) idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { return err } s.index[h] = idxf return err } func (s *ObjectStorage) NewEncodedObject() plumbing.EncodedObject { return &plumbing.MemoryObject{} } func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { if err := s.requireIndex(); err != nil { return nil, err } w, err := s.dir.NewObjectPack() if err != nil { return nil, err } w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) { index, err := writer.Index() if err == nil { s.index[h] = index } } return w, nil } // SetEncodedObject adds a new object to the storage. func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (h plumbing.Hash, err error) { if o.Type() == plumbing.OFSDeltaObject || o.Type() == plumbing.REFDeltaObject { return plumbing.ZeroHash, plumbing.ErrInvalidType } ow, err := s.dir.NewObject() if err != nil { return plumbing.ZeroHash, err } defer ioutil.CheckClose(ow, &err) or, err := o.Reader() if err != nil { return plumbing.ZeroHash, err } defer ioutil.CheckClose(or, &err) if err = ow.WriteHeader(o.Type(), o.Size()); err != nil { return plumbing.ZeroHash, err } if _, err = io.Copy(ow, or); err != nil { return plumbing.ZeroHash, err } return o.Hash(), err } // HasEncodedObject returns nil if the object exists, without actually // reading the object data from storage. func (s *ObjectStorage) HasEncodedObject(h plumbing.Hash) (err error) { // Check unpacked objects f, err := s.dir.Object(h) if err != nil { if !os.IsNotExist(err) { return err } // Fall through to check packed objects. } else { defer ioutil.CheckClose(f, &err) return nil } // Check packed objects. if err := s.requireIndex(); err != nil { return err } _, _, offset := s.findObjectInPackfile(h) if offset == -1 { return plumbing.ErrObjectNotFound } return nil } func (s *ObjectStorage) encodedObjectSizeFromUnpacked(h plumbing.Hash) ( size int64, err error) { f, err := s.dir.Object(h) if err != nil { if os.IsNotExist(err) { return 0, plumbing.ErrObjectNotFound } return 0, err } r, err := objfile.NewReader(f) if err != nil { return 0, err } defer ioutil.CheckClose(r, &err) _, size, err = r.Header() return size, err } func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( size int64, err error) { if err := s.requireIndex(); err != nil { return 0, err } pack, _, offset := s.findObjectInPackfile(h) if offset == -1 { return 0, plumbing.ErrObjectNotFound } f, err := s.dir.ObjectPack(pack) if err != nil { return 0, err } defer ioutil.CheckClose(f, &err) idx := s.index[pack] hash, err := idx.FindHash(offset) if err == nil { obj, ok := s.objectCache.Get(hash) if ok { return obj.Size(), nil } } else if err != nil && err != plumbing.ErrObjectNotFound { return 0, err } var p *packfile.Packfile if s.objectCache != nil { p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) } else { p = packfile.NewPackfile(idx, s.dir.Fs(), f) } return p.GetSizeByOffset(offset) } // EncodedObjectSize returns the plaintext size of the given object, // without actually reading the full object data from storage. func (s *ObjectStorage) EncodedObjectSize(h plumbing.Hash) ( size int64, err error) { size, err = s.encodedObjectSizeFromUnpacked(h) if err != nil && err != plumbing.ErrObjectNotFound { return 0, err } else if err == nil { return size, nil } return s.encodedObjectSizeFromPackfile(h) } // EncodedObject returns the object with the given hash, by searching for it in // the packfile and the git object directories. func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { var obj plumbing.EncodedObject var err error if s.index != nil { obj, err = s.getFromPackfile(h, false) if err == plumbing.ErrObjectNotFound { obj, err = s.getFromUnpacked(h) } } else { obj, err = s.getFromUnpacked(h) if err == plumbing.ErrObjectNotFound { obj, err = s.getFromPackfile(h, false) } } // If the error is still object not found, check if it's a shared object // repository. if err == plumbing.ErrObjectNotFound { dotgits, e := s.dir.Alternates() if e == nil { // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { o := NewObjectStorage(dg, s.objectCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue } return enobj, nil } } } if err != nil { return nil, err } if plumbing.AnyObject != t && obj.Type() != t { return nil, plumbing.ErrObjectNotFound } return obj, nil } // DeltaObject returns the object with the given hash, by searching for // it in the packfile and the git object directories. func (s *ObjectStorage) DeltaObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { obj, err := s.getFromUnpacked(h) if err == plumbing.ErrObjectNotFound { obj, err = s.getFromPackfile(h, true) } if err != nil { return nil, err } if plumbing.AnyObject != t && obj.Type() != t { return nil, plumbing.ErrObjectNotFound } return obj, nil } func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { f, err := s.dir.Object(h) if err != nil { if os.IsNotExist(err) { return nil, plumbing.ErrObjectNotFound } return nil, err } defer ioutil.CheckClose(f, &err) if cacheObj, found := s.objectCache.Get(h); found { return cacheObj, nil } obj = s.NewEncodedObject() r, err := objfile.NewReader(f) if err != nil { return nil, err } defer ioutil.CheckClose(r, &err) t, size, err := r.Header() if err != nil { return nil, err } obj.SetType(t) obj.SetSize(size) w, err := obj.Writer() if err != nil { return nil, err } s.objectCache.Put(obj) _, err = io.Copy(w, r) return obj, err } // Get returns the object with the given hash, by searching for it in // the packfile. func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( plumbing.EncodedObject, error) { if err := s.requireIndex(); err != nil { return nil, err } pack, hash, offset := s.findObjectInPackfile(h) if offset == -1 { return nil, plumbing.ErrObjectNotFound } f, err := s.dir.ObjectPack(pack) if err != nil { return nil, err } if !s.options.KeepDescriptors { defer ioutil.CheckClose(f, &err) } idx := s.index[pack] if canBeDelta { return s.decodeDeltaObjectAt(f, idx, offset, hash) } return s.decodeObjectAt(f, idx, offset) } func (s *ObjectStorage) decodeObjectAt( f billy.File, idx idxfile.Index, offset int64, ) (plumbing.EncodedObject, error) { hash, err := idx.FindHash(offset) if err == nil { obj, ok := s.objectCache.Get(hash) if ok { return obj, nil } } if err != nil && err != plumbing.ErrObjectNotFound { return nil, err } var p *packfile.Packfile if s.objectCache != nil { p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) } else { p = packfile.NewPackfile(idx, s.dir.Fs(), f) } return p.GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, idx idxfile.Index, offset int64, hash plumbing.Hash, ) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err } p := packfile.NewScanner(f) header, err := p.SeekObjectHeader(offset) if err != nil { return nil, err } var ( base plumbing.Hash ) switch header.Type { case plumbing.REFDeltaObject: base = header.Reference case plumbing.OFSDeltaObject: base, err = idx.FindHash(header.OffsetReference) if err != nil { return nil, err } default: return s.decodeObjectAt(f, idx, offset) } obj := &plumbing.MemoryObject{} obj.SetType(header.Type) w, err := obj.Writer() if err != nil { return nil, err } if _, _, err := p.NextObject(w); err != nil { return nil, err } return newDeltaObject(obj, hash, base, header.Length), nil } func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) { for packfile, index := range s.index { offset, err := index.FindOffset(h) if err == nil { return packfile, h, offset } } return plumbing.ZeroHash, plumbing.ZeroHash, -1 } // IterEncodedObjects returns an iterator for all the objects in the packfile // with the given type. func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.EncodedObjectIter, error) { objects, err := s.dir.Objects() if err != nil { return nil, err } seen := make(map[plumbing.Hash]struct{}) var iters []storer.EncodedObjectIter if len(objects) != 0 { iters = append(iters, &objectsIter{s: s, t: t, h: objects}) seen = hashListAsMap(objects) } packi, err := s.buildPackfileIters(t, seen) if err != nil { return nil, err } iters = append(iters, packi) return storer.NewMultiEncodedObjectIter(iters), nil } func (s *ObjectStorage) buildPackfileIters( t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, ) (storer.EncodedObjectIter, error) { if err := s.requireIndex(); err != nil { return nil, err } packs, err := s.dir.ObjectPacks() if err != nil { return nil, err } return &lazyPackfilesIter{ hashes: packs, open: func(h plumbing.Hash) (storer.EncodedObjectIter, error) { pack, err := s.dir.ObjectPack(h) if err != nil { return nil, err } return newPackfileIter( s.dir.Fs(), pack, t, seen, s.index[h], s.objectCache, s.options.KeepDescriptors, ) }, }, nil } // Close closes all opened files. func (s *ObjectStorage) Close() error { return s.dir.Close() } type lazyPackfilesIter struct { hashes []plumbing.Hash open func(h plumbing.Hash) (storer.EncodedObjectIter, error) cur storer.EncodedObjectIter } func (it *lazyPackfilesIter) Next() (plumbing.EncodedObject, error) { for { if it.cur == nil { if len(it.hashes) == 0 { return nil, io.EOF } h := it.hashes[0] it.hashes = it.hashes[1:] sub, err := it.open(h) if err == io.EOF { continue } else if err != nil { return nil, err } it.cur = sub } ob, err := it.cur.Next() if err == io.EOF { it.cur.Close() it.cur = nil continue } else if err != nil { return nil, err } return ob, nil } } func (it *lazyPackfilesIter) ForEach(cb func(plumbing.EncodedObject) error) error { return storer.ForEachIterator(it, cb) } func (it *lazyPackfilesIter) Close() { if it.cur != nil { it.cur.Close() it.cur = nil } it.hashes = nil } type packfileIter struct { pack billy.File iter storer.EncodedObjectIter seen map[plumbing.Hash]struct{} // tells whether the pack file should be left open after iteration or not keepPack bool } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile // and object type. Packfile and index file will be closed after they're // used. If keepPack is true the packfile won't be closed after the iteration // finished. func NewPackfileIter( fs billy.Filesystem, f billy.File, idxFile billy.File, t plumbing.ObjectType, keepPack bool, ) (storer.EncodedObjectIter, error) { idx := idxfile.NewMemoryIndex() if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { return nil, err } if err := idxFile.Close(); err != nil { return nil, err } seen := make(map[plumbing.Hash]struct{}) return newPackfileIter(fs, f, t, seen, idx, nil, keepPack) } func newPackfileIter( fs billy.Filesystem, f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, index idxfile.Index, cache cache.Object, keepPack bool, ) (storer.EncodedObjectIter, error) { var p *packfile.Packfile if cache != nil { p = packfile.NewPackfileWithCache(index, fs, f, cache) } else { p = packfile.NewPackfile(index, fs, f) } iter, err := p.GetByType(t) if err != nil { return nil, err } return &packfileIter{ pack: f, iter: iter, seen: seen, keepPack: keepPack, }, nil } func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { for { obj, err := iter.iter.Next() if err != nil { return nil, err } if _, ok := iter.seen[obj.Hash()]; ok { continue } return obj, nil } } func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { for { o, err := iter.Next() if err != nil { if err == io.EOF { iter.Close() return nil } return err } if err := cb(o); err != nil { return err } } } func (iter *packfileIter) Close() { iter.iter.Close() if !iter.keepPack { _ = iter.pack.Close() } } type objectsIter struct { s *ObjectStorage t plumbing.ObjectType h []plumbing.Hash } func (iter *objectsIter) Next() (plumbing.EncodedObject, error) { if len(iter.h) == 0 { return nil, io.EOF } obj, err := iter.s.getFromUnpacked(iter.h[0]) iter.h = iter.h[1:] if err != nil { return nil, err } if iter.t != plumbing.AnyObject && iter.t != obj.Type() { return iter.Next() } return obj, err } func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error { for { o, err := iter.Next() if err != nil { if err == io.EOF { return nil } return err } if err := cb(o); err != nil { return err } } } func (iter *objectsIter) Close() { iter.h = []plumbing.Hash{} } func hashListAsMap(l []plumbing.Hash) map[plumbing.Hash]struct{} { m := make(map[plumbing.Hash]struct{}, len(l)) for _, h := range l { m[h] = struct{}{} } return m } func (s *ObjectStorage) ForEachObjectHash(fun func(plumbing.Hash) error) error { err := s.dir.ForEachObjectHash(fun) if err == storer.ErrStop { return nil } return err } func (s *ObjectStorage) LooseObjectTime(hash plumbing.Hash) (time.Time, error) { fi, err := s.dir.ObjectStat(hash) if err != nil { return time.Time{}, err } return fi.ModTime(), nil } func (s *ObjectStorage) DeleteLooseObject(hash plumbing.Hash) error { return s.dir.ObjectDelete(hash) } func (s *ObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { return s.dir.ObjectPacks() } func (s *ObjectStorage) DeleteOldObjectPackAndIndex(h plumbing.Hash, t time.Time) error { return s.dir.DeleteOldObjectPackAndIndex(h, t) }