Fix package upload for files >32mb (#20622) (#20635)

* Rewind file before first read.
* Added tests.

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
KN4CK3R 2022-08-03 00:45:38 +02:00 committed by GitHub
parent 0747592865
commit bd2218e14c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 98 additions and 5 deletions

View File

@ -0,0 +1,47 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package packages
import (
"fmt"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestHashedBuffer(t *testing.T) {
cases := []struct {
MaxMemorySize int
Data string
HashMD5 string
HashSHA1 string
HashSHA256 string
HashSHA512 string
}{
{5, "test", "098f6bcd4621d373cade4e832627b4f6", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"},
{5, "testtest", "05a671c66aefea124cc08b76ea6d30bb", "51abb9636078defbf888d8457a7c76f85c8f114c", "37268335dd6931045bdcdf92623ff819a64244b53d0e746d438797349d4da578", "125d6d03b32c84d492747f79cf0bf6e179d287f341384eb5d6d3197525ad6be8e6df0116032935698f99a09e265073d1d6c32c274591bf1d0a20ad67cba921bc"},
}
for _, c := range cases {
buf, err := CreateHashedBufferFromReader(strings.NewReader(c.Data), c.MaxMemorySize)
assert.NoError(t, err)
assert.EqualValues(t, len(c.Data), buf.Size())
data, err := io.ReadAll(buf)
assert.NoError(t, err)
assert.Equal(t, c.Data, string(data))
hashMD5, hashSHA1, hashSHA256, hashSHA512 := buf.Sums()
assert.Equal(t, c.HashMD5, fmt.Sprintf("%x", hashMD5))
assert.Equal(t, c.HashSHA1, fmt.Sprintf("%x", hashSHA1))
assert.Equal(t, c.HashSHA256, fmt.Sprintf("%x", hashSHA256))
assert.Equal(t, c.HashSHA512, fmt.Sprintf("%x", hashSHA512))
assert.NoError(t, buf.Close())
}
}

View File

@ -103,35 +103,45 @@ func (b *FileBackedBuffer) Size() int64 {
return b.size return b.size
} }
func (b *FileBackedBuffer) switchToReader() { func (b *FileBackedBuffer) switchToReader() error {
if b.reader != nil { if b.reader != nil {
return return nil
} }
if b.file != nil { if b.file != nil {
if _, err := b.file.Seek(0, io.SeekStart); err != nil {
return err
}
b.reader = b.file b.reader = b.file
} else { } else {
b.reader = bytes.NewReader(b.buffer.Bytes()) b.reader = bytes.NewReader(b.buffer.Bytes())
} }
return nil
} }
// Read implements io.Reader // Read implements io.Reader
func (b *FileBackedBuffer) Read(p []byte) (int, error) { func (b *FileBackedBuffer) Read(p []byte) (int, error) {
b.switchToReader() if err := b.switchToReader(); err != nil {
return 0, err
}
return b.reader.Read(p) return b.reader.Read(p)
} }
// ReadAt implements io.ReaderAt // ReadAt implements io.ReaderAt
func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) { func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) {
b.switchToReader() if err := b.switchToReader(); err != nil {
return 0, err
}
return b.reader.ReadAt(p, off) return b.reader.ReadAt(p, off)
} }
// Seek implements io.Seeker // Seek implements io.Seeker
func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) {
b.switchToReader() if err := b.switchToReader(); err != nil {
return 0, err
}
return b.reader.Seek(offset, whence) return b.reader.Seek(offset, whence)
} }

View File

@ -0,0 +1,36 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package filebuffer
import (
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFileBackedBuffer(t *testing.T) {
cases := []struct {
MaxMemorySize int
Data string
}{
{5, "test"},
{5, "testtest"},
}
for _, c := range cases {
buf, err := CreateFromReader(strings.NewReader(c.Data), c.MaxMemorySize)
assert.NoError(t, err)
assert.EqualValues(t, len(c.Data), buf.Size())
data, err := io.ReadAll(buf)
assert.NoError(t, err)
assert.Equal(t, c.Data, string(data))
assert.NoError(t, buf.Close())
}
}