From 631539c104c790fccf31a80a92e31fcc8a16521c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 2 Aug 2022 15:35:48 +0200 Subject: [PATCH] Fix package upload for files >32mb (#20622) * Rewind file before first read. * Added tests. Co-authored-by: Lunny Xiao --- modules/packages/hashed_buffer_test.go | 47 +++++++++++++++++++ modules/util/filebuffer/file_backed_buffer.go | 20 ++++++-- .../filebuffer/file_backed_buffer_test.go | 36 ++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 modules/packages/hashed_buffer_test.go create mode 100644 modules/util/filebuffer/file_backed_buffer_test.go diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go new file mode 100644 index 0000000000..e21ec67e1f --- /dev/null +++ b/modules/packages/hashed_buffer_test.go @@ -0,0 +1,47 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHashedBuffer(t *testing.T) { + cases := []struct { + MaxMemorySize int + Data string + HashMD5 string + HashSHA1 string + HashSHA256 string + HashSHA512 string + }{ + {5, "test", "098f6bcd4621d373cade4e832627b4f6", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"}, + {5, "testtest", "05a671c66aefea124cc08b76ea6d30bb", "51abb9636078defbf888d8457a7c76f85c8f114c", "37268335dd6931045bdcdf92623ff819a64244b53d0e746d438797349d4da578", "125d6d03b32c84d492747f79cf0bf6e179d287f341384eb5d6d3197525ad6be8e6df0116032935698f99a09e265073d1d6c32c274591bf1d0a20ad67cba921bc"}, + } + + for _, c := range cases { + buf, err := CreateHashedBufferFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + assert.NoError(t, err) + + assert.EqualValues(t, len(c.Data), buf.Size()) + + data, err := io.ReadAll(buf) + assert.NoError(t, err) + assert.Equal(t, c.Data, string(data)) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := buf.Sums() + assert.Equal(t, c.HashMD5, fmt.Sprintf("%x", hashMD5)) + assert.Equal(t, c.HashSHA1, fmt.Sprintf("%x", hashSHA1)) + assert.Equal(t, c.HashSHA256, fmt.Sprintf("%x", hashSHA256)) + assert.Equal(t, c.HashSHA512, fmt.Sprintf("%x", hashSHA512)) + + assert.NoError(t, buf.Close()) + } +} diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 128030b4c5..8e3e138e04 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -103,35 +103,45 @@ func (b *FileBackedBuffer) Size() int64 { return b.size } -func (b *FileBackedBuffer) switchToReader() { +func (b *FileBackedBuffer) switchToReader() error { if b.reader != nil { - return + return nil } if b.file != nil { + if _, err := b.file.Seek(0, io.SeekStart); err != nil { + return err + } b.reader = b.file } else { b.reader = bytes.NewReader(b.buffer.Bytes()) } + return nil } // Read implements io.Reader func (b *FileBackedBuffer) Read(p []byte) (int, error) { - b.switchToReader() + if err := b.switchToReader(); err != nil { + return 0, err + } return b.reader.Read(p) } // ReadAt implements io.ReaderAt func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) { - b.switchToReader() + if err := b.switchToReader(); err != nil { + return 0, err + } return b.reader.ReadAt(p, off) } // Seek implements io.Seeker func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { - b.switchToReader() + if err := b.switchToReader(); err != nil { + return 0, err + } return b.reader.Seek(offset, whence) } diff --git a/modules/util/filebuffer/file_backed_buffer_test.go b/modules/util/filebuffer/file_backed_buffer_test.go new file mode 100644 index 0000000000..83ef58561d --- /dev/null +++ b/modules/util/filebuffer/file_backed_buffer_test.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package filebuffer + +import ( + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFileBackedBuffer(t *testing.T) { + cases := []struct { + MaxMemorySize int + Data string + }{ + {5, "test"}, + {5, "testtest"}, + } + + for _, c := range cases { + buf, err := CreateFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + assert.NoError(t, err) + + assert.EqualValues(t, len(c.Data), buf.Size()) + + data, err := io.ReadAll(buf) + assert.NoError(t, err) + assert.Equal(t, c.Data, string(data)) + + assert.NoError(t, buf.Close()) + } +}