mirror of https://github.com/go-gitea/gitea.git
update: macaron cores,gzip,session (#10522)
Co-authored-by: zeripath <art27@cantab.net>pull/10531/head
6543
GitHub
parent
694f44660f
commit
8d2059a201
13
go.mod
13
go.mod
|
|
@ -8,13 +8,13 @@ require (
|
|||
gitea.com/macaron/binding v0.0.0-20190822013154-a5f53841ed2b
|
||||
gitea.com/macaron/cache v0.0.0-20190822004001-a6e7fee4ee76
|
||||
gitea.com/macaron/captcha v0.0.0-20190822015246-daa973478bae
|
||||
gitea.com/macaron/cors v0.0.0-20190821152825-7dcef4a17175
|
||||
gitea.com/macaron/cors v0.0.0-20190826180238-95aec09ea8b4
|
||||
gitea.com/macaron/csrf v0.0.0-20190822024205-3dc5a4474439
|
||||
gitea.com/macaron/gzip v0.0.0-20191118033930-0c4c5566a0e5
|
||||
gitea.com/macaron/gzip v0.0.0-20191118041502-506895b47aae
|
||||
gitea.com/macaron/i18n v0.0.0-20190822004228-474e714e2223
|
||||
gitea.com/macaron/inject v0.0.0-20190805023432-d4c86e31027a
|
||||
gitea.com/macaron/macaron v1.4.0
|
||||
gitea.com/macaron/session v0.0.0-20190821211443-122c47c5f705
|
||||
gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d
|
||||
gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7
|
||||
github.com/PuerkitoBio/goquery v1.5.0
|
||||
github.com/RoaringBitmap/roaring v0.4.21 // indirect
|
||||
|
|
@ -25,7 +25,6 @@ require (
|
|||
github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f // indirect
|
||||
github.com/boombuler/barcode v0.0.0-20161226211916-fe0f26ff6d26 // indirect
|
||||
github.com/couchbase/gomemcached v0.0.0-20191004160342-7b5da2ec40b2 // indirect
|
||||
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 // indirect
|
||||
github.com/couchbase/vellum v0.0.0-20190829182332-ef2e028c01fd // indirect
|
||||
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
|
|
@ -59,7 +58,7 @@ require (
|
|||
github.com/joho/godotenv v1.3.0 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657
|
||||
github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4
|
||||
github.com/klauspost/compress v1.9.2
|
||||
github.com/klauspost/compress v1.10.2
|
||||
github.com/lafriks/xormstore v1.3.2
|
||||
github.com/lib/pq v1.2.0
|
||||
github.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96
|
||||
|
|
@ -99,7 +98,7 @@ require (
|
|||
github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53
|
||||
github.com/yuin/goldmark v1.1.19
|
||||
go.etcd.io/bbolt v1.3.3 // indirect
|
||||
golang.org/x/crypto v0.0.0-20200219234226-1ad67e1f0ef4
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
|
||||
golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c
|
||||
|
|
@ -108,7 +107,7 @@ require (
|
|||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect
|
||||
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
|
||||
gopkg.in/ini.v1 v1.51.1
|
||||
gopkg.in/ini.v1 v1.52.0
|
||||
gopkg.in/ldap.v3 v3.0.2
|
||||
gopkg.in/src-d/go-billy.v4 v4.3.2
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1
|
||||
|
|
|
|||
20
go.sum
20
go.sum
|
|
@ -17,12 +17,12 @@ gitea.com/macaron/cache v0.0.0-20190822004001-a6e7fee4ee76 h1:mMsMEg90c5KXQgRWsH
|
|||
gitea.com/macaron/cache v0.0.0-20190822004001-a6e7fee4ee76/go.mod h1:NFHb9Of+LUnU86bU20CiXXg6ZlgCJ4XytP14UsHOXFs=
|
||||
gitea.com/macaron/captcha v0.0.0-20190822015246-daa973478bae h1:9C31eOCpMPbW9rDVq8M1UJ+5HZVYA38HHaKCVcRYDpI=
|
||||
gitea.com/macaron/captcha v0.0.0-20190822015246-daa973478bae/go.mod h1:J5h3N+1nKTXtU1x4GxexaQKgAz8UiWecNwi/CfX7CtQ=
|
||||
gitea.com/macaron/cors v0.0.0-20190821152825-7dcef4a17175 h1:ikzdAGB6SsUGByW5wKlK+JwzfgQHX+GJnBwEfsaCTNY=
|
||||
gitea.com/macaron/cors v0.0.0-20190821152825-7dcef4a17175/go.mod h1:rtOK4J20kpMD9XcNsnO5YA843YSTe/MUMbDj/TJ/Q7A=
|
||||
gitea.com/macaron/cors v0.0.0-20190826180238-95aec09ea8b4 h1:e2rAFDejB0qN8OrY4xP4XSu8/yT6QmWxDZpB3J7r2GU=
|
||||
gitea.com/macaron/cors v0.0.0-20190826180238-95aec09ea8b4/go.mod h1:rtOK4J20kpMD9XcNsnO5YA843YSTe/MUMbDj/TJ/Q7A=
|
||||
gitea.com/macaron/csrf v0.0.0-20190822024205-3dc5a4474439 h1:88c34YM29a1GlWLrLBaG/GTT2htDdJz1u3n9+lmPolg=
|
||||
gitea.com/macaron/csrf v0.0.0-20190822024205-3dc5a4474439/go.mod h1:IsQPHx73HnnqFBYiVHjg87q4XBZyGXXu77xANukvZuk=
|
||||
gitea.com/macaron/gzip v0.0.0-20191118033930-0c4c5566a0e5 h1:G/a7r0r2jEelSynBlv1+PAEZQKfsdRHQUMb1PlNvemM=
|
||||
gitea.com/macaron/gzip v0.0.0-20191118033930-0c4c5566a0e5/go.mod h1:jGHtoovArcQj+sw7NJxyPgjuRxOSG9a/oFu3VkLRTKQ=
|
||||
gitea.com/macaron/gzip v0.0.0-20191118041502-506895b47aae h1:OXxYwGmGNfYrC0/sUUL9KSvr2Sfvmzwgd2YD65vIjGE=
|
||||
gitea.com/macaron/gzip v0.0.0-20191118041502-506895b47aae/go.mod h1:jGHtoovArcQj+sw7NJxyPgjuRxOSG9a/oFu3VkLRTKQ=
|
||||
gitea.com/macaron/i18n v0.0.0-20190822004228-474e714e2223 h1:iZWwQif/LHMjBgfY/ua8CFVa4XMDfbbs7EZ0Q1dYguU=
|
||||
gitea.com/macaron/i18n v0.0.0-20190822004228-474e714e2223/go.mod h1:+qsc10s4hBsHKU/9luGGumFh4m5FFVc7uih+8/mM1NY=
|
||||
gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591/go.mod h1:h6E4kLao1Yko6DOU6QDnQPcuoNzvbZqzj2mtPcEn1aM=
|
||||
|
|
@ -35,6 +35,8 @@ gitea.com/macaron/macaron v1.4.0 h1:FY1QDGqyuUzs21K6ChkbYbRUfwL7v2aUrhNEJ0IgsAw=
|
|||
gitea.com/macaron/macaron v1.4.0/go.mod h1:P7hfDbQjcW22lkYkXlxdRIfWOXxH2+K4EogN4Q0UlLY=
|
||||
gitea.com/macaron/session v0.0.0-20190821211443-122c47c5f705 h1:mvkQGAlON1Z6Y8pqa/+FpYIskk54mazuECUfZK5oTg0=
|
||||
gitea.com/macaron/session v0.0.0-20190821211443-122c47c5f705/go.mod h1:1ujH0jD6Ca4iK9NL0Q2a7fG2chvXx5hVa7hBfABwpkA=
|
||||
gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d h1:XLww3CvnFZkXVwauN67fniDaIpIqsE+9KVcxlZKlvLU=
|
||||
gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d/go.mod h1:FanKy3WjWb5iw/iZBPk4ggoQT9FcM6bkBPvmDmsH6tY=
|
||||
gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 h1:N9QFoeNsUXLhl14mefLzGluqV7w2mGU3u+iZU+jCeWk=
|
||||
gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7/go.mod h1:kgsbFPPS4P+acDYDOPDa3N4IWWOuDJt5/INKRUz7aks=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
|
|
@ -349,6 +351,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
|
|||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
|
||||
github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.10.2 h1:Znfn6hXZAHaLPNnlqUYRrBSReFHYybslgv4PTiyz6P0=
|
||||
github.com/klauspost/compress v1.10.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
|
|
@ -602,8 +606,8 @@ golang.org/x/crypto v0.0.0-20190907121410-71b5226ff739/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad h1:5E5raQxcv+6CZ11RrBYQe5WRbUIWpScjh0kvHZkZIrQ=
|
||||
golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200219234226-1ad67e1f0ef4 h1:4icQlpeqbz3WxfgP6Eq3szTj95KTrlH/CwzBzoxuFd0=
|
||||
golang.org/x/crypto v0.0.0-20200219234226-1ad67e1f0ef4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
|
|
@ -746,8 +750,8 @@ gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
|||
gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.44.2/go.mod h1:M3Cogqpuv0QCi3ExAY5V4uOt4qb/R3xZubo9m8lK5wg=
|
||||
gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho=
|
||||
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4=
|
||||
gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ldap.v3 v3.0.2 h1:R6RBtabK6e1GO0eQKtkyOFbAHO73QesLzI2w2DZ6b9w=
|
||||
gopkg.in/ldap.v3 v3.0.2/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
|
|
|
|||
|
|
@ -151,7 +151,8 @@ func CORS(options ...Options) macaron.Handler {
|
|||
}
|
||||
})
|
||||
if reqOptions {
|
||||
ctx.Status(200) // return response
|
||||
ctx.Resp.WriteHeader(200) // return response
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
kind: pipeline
|
||||
name: go1-1-2
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.12
|
||||
environment:
|
||||
GOPROXY: https://goproxy.cn
|
||||
commands:
|
||||
- go build -v
|
||||
- go test -v -race -coverprofile=coverage.txt -covermode=atomic
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
name: go1-1-3
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.13
|
||||
environment:
|
||||
GOPROXY: https://goproxy.cn
|
||||
commands:
|
||||
- go build -v
|
||||
- go test -v -race -coverprofile=coverage.txt -covermode=atomic
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# gzip
|
||||
|
||||
Middleware gzip provides gzip comparess middleware for [Macaron](https://gitea.com/macaron/macaron).
|
||||
|
||||
### Installation
|
||||
|
||||
go get gitea.com/macaron/gzip
|
||||
|
||||
## Getting Help
|
||||
|
||||
- [API Reference](https://godoc.org/gitea.com/macaron/gzip)
|
||||
|
||||
## Credits
|
||||
|
||||
This package is a modified version of [go-macaron gzip](github.com/go-macaron/gzip).
|
||||
|
||||
## License
|
||||
|
||||
This project is under the Apache License, Version 2.0. See the [LICENSE](LICENSE) file for the full license text.
|
||||
|
|
@ -6,7 +6,7 @@ require (
|
|||
gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb
|
||||
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
|
||||
github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d // indirect
|
||||
github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b // indirect
|
||||
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 // indirect
|
||||
github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7
|
||||
github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 // indirect
|
||||
github.com/edsrzf/mmap-go v1.0.0 // indirect
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK
|
|||
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
|
||||
github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d h1:XMf4E1U+b9E3ElF0mjvfXZdflBRZz4gLp16nQ/QSHQM=
|
||||
github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
|
||||
github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b h1:bZ9rKU2/V8sY+NulSfxDOnXTWcs1rySqdF1sVepihvo=
|
||||
github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
|
||||
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 h1:0WMIDtuXCKEm4wtAJgAAXa/qtM5O9MariLwgHaRlYmk=
|
||||
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
|
||||
github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7 h1:1XjEY/gnjQ+AfXef2U6dxCquhiRzkEpxZuWqs+QxTL8=
|
||||
github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7/go.mod h1:mby/05p8HE5yHEAKiIH/555NoblMs7PtW6NrYshDruc=
|
||||
github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ=
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ const (
|
|||
maxHashOffset = 1 << 24
|
||||
|
||||
skipNever = math.MaxInt32
|
||||
|
||||
debugDeflate = false
|
||||
)
|
||||
|
||||
type compressionLevel struct {
|
||||
|
|
@ -59,15 +61,13 @@ type compressionLevel struct {
|
|||
// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
|
||||
var levels = []compressionLevel{
|
||||
{}, // 0
|
||||
// Level 1-4 uses specialized algorithm - values not used
|
||||
// Level 1-6 uses specialized algorithm - values not used
|
||||
{0, 0, 0, 0, 0, 1},
|
||||
{0, 0, 0, 0, 0, 2},
|
||||
{0, 0, 0, 0, 0, 3},
|
||||
{0, 0, 0, 0, 0, 4},
|
||||
// For levels 5-6 we don't bother trying with lazy matches.
|
||||
// Lazy matching is at least 30% slower, with 1.5% increase.
|
||||
{6, 0, 12, 8, 12, 5},
|
||||
{8, 0, 24, 16, 16, 6},
|
||||
{0, 0, 0, 0, 0, 5},
|
||||
{0, 0, 0, 0, 0, 6},
|
||||
// Levels 7-9 use increasingly more lazy matching
|
||||
// and increasingly stringent conditions for "good enough".
|
||||
{8, 8, 24, 16, skipNever, 7},
|
||||
|
|
@ -203,9 +203,8 @@ func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error {
|
|||
// This is much faster than doing a full encode.
|
||||
// Should only be used after a start/reset.
|
||||
func (d *compressor) fillWindow(b []byte) {
|
||||
// Do not fill window if we are in store-only mode,
|
||||
// use constant or Snappy compression.
|
||||
if d.level == 0 {
|
||||
// Do not fill window if we are in store-only or huffman mode.
|
||||
if d.level <= 0 {
|
||||
return
|
||||
}
|
||||
if d.fast != nil {
|
||||
|
|
@ -368,7 +367,7 @@ func (d *compressor) deflateLazy() {
|
|||
// Sanity enables additional runtime tests.
|
||||
// It's intended to be used during development
|
||||
// to supplement the currently ad-hoc unit tests.
|
||||
const sanity = false
|
||||
const sanity = debugDeflate
|
||||
|
||||
if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
|
||||
return
|
||||
|
|
@ -644,7 +643,7 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
|
|||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).store
|
||||
case level == ConstantCompression:
|
||||
d.w.logReusePenalty = uint(4)
|
||||
d.w.logNewTablePenalty = 4
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeHuff
|
||||
|
|
@ -652,13 +651,13 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
|
|||
level = 5
|
||||
fallthrough
|
||||
case level >= 1 && level <= 6:
|
||||
d.w.logReusePenalty = uint(level + 1)
|
||||
d.w.logNewTablePenalty = 6
|
||||
d.fast = newFastEnc(level)
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeFast
|
||||
case 7 <= level && level <= 9:
|
||||
d.w.logReusePenalty = uint(level)
|
||||
d.w.logNewTablePenalty = 10
|
||||
d.state = &advancedState{}
|
||||
d.compressionLevel = levels[level]
|
||||
d.initDeflate()
|
||||
|
|
@ -667,6 +666,7 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
|
|||
default:
|
||||
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
|
||||
}
|
||||
d.level = level
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -720,6 +720,7 @@ func (d *compressor) close() error {
|
|||
return d.w.err
|
||||
}
|
||||
d.w.flush()
|
||||
d.w.reset(nil)
|
||||
return d.w.err
|
||||
}
|
||||
|
||||
|
|
@ -750,8 +751,7 @@ func NewWriter(w io.Writer, level int) (*Writer, error) {
|
|||
// can only be decompressed by a Reader initialized with the
|
||||
// same dictionary.
|
||||
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
|
||||
dw := &dictWriter{w}
|
||||
zw, err := NewWriter(dw, level)
|
||||
zw, err := NewWriter(w, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -760,14 +760,6 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
|
|||
return zw, err
|
||||
}
|
||||
|
||||
type dictWriter struct {
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (w *dictWriter) Write(b []byte) (n int, err error) {
|
||||
return w.w.Write(b)
|
||||
}
|
||||
|
||||
// A Writer takes data written to it and writes the compressed
|
||||
// form of that data to an underlying writer (see NewWriter).
|
||||
type Writer struct {
|
||||
|
|
@ -805,11 +797,12 @@ func (w *Writer) Close() error {
|
|||
// the result of NewWriter or NewWriterDict called with dst
|
||||
// and w's level and dictionary.
|
||||
func (w *Writer) Reset(dst io.Writer) {
|
||||
if dw, ok := w.d.w.writer.(*dictWriter); ok {
|
||||
if len(w.dict) > 0 {
|
||||
// w was created with NewWriterDict
|
||||
dw.w = dst
|
||||
w.d.reset(dw)
|
||||
w.d.fillWindow(w.dict)
|
||||
w.d.reset(dst)
|
||||
if dst != nil {
|
||||
w.d.fillWindow(w.dict)
|
||||
}
|
||||
} else {
|
||||
// w was created with NewWriter
|
||||
w.d.reset(dst)
|
||||
|
|
|
|||
|
|
@ -35,17 +35,17 @@ func newFastEnc(level int) fastEnc {
|
|||
}
|
||||
|
||||
const (
|
||||
tableBits = 16 // Bits used in the table
|
||||
tableBits = 15 // Bits used in the table
|
||||
tableSize = 1 << tableBits // Size of the table
|
||||
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
|
||||
baseMatchOffset = 1 // The smallest match offset
|
||||
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
|
||||
maxMatchOffset = 1 << 15 // The largest match offset
|
||||
|
||||
bTableBits = 18 // Bits used in the big tables
|
||||
bTableSize = 1 << bTableBits // Size of the table
|
||||
allocHistory = maxMatchOffset * 10 // Size to preallocate for history.
|
||||
bufferReset = (1 << 31) - allocHistory - maxStoreBlockSize // Reset the buffer offset when reaching this.
|
||||
bTableBits = 17 // Bits used in the big tables
|
||||
bTableSize = 1 << bTableBits // Size of the table
|
||||
allocHistory = maxStoreBlockSize * 10 // Size to preallocate for history.
|
||||
bufferReset = (1 << 31) - allocHistory - maxStoreBlockSize - 1 // Reset the buffer offset when reaching this.
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -92,7 +92,6 @@ func hash(u uint32) uint32 {
|
|||
}
|
||||
|
||||
type tableEntry struct {
|
||||
val uint32
|
||||
offset int32
|
||||
}
|
||||
|
||||
|
|
@ -210,16 +209,14 @@ func (e *fastGen) matchlenLong(s, t int32, src []byte) int32 {
|
|||
|
||||
// Reset the encoding table.
|
||||
func (e *fastGen) Reset() {
|
||||
if cap(e.hist) < int(maxMatchOffset*8) {
|
||||
l := maxMatchOffset * 8
|
||||
// Make it at least 1MB.
|
||||
if l < 1<<20 {
|
||||
l = 1 << 20
|
||||
}
|
||||
e.hist = make([]byte, 0, l)
|
||||
if cap(e.hist) < allocHistory {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
}
|
||||
// We offset current position so everything will be out of reach.
|
||||
// If we are above the buffer reset it will be cleared anyway since len(hist) == 0.
|
||||
if e.cur <= bufferReset {
|
||||
e.cur += maxMatchOffset + int32(len(e.hist))
|
||||
}
|
||||
// We offset current position so everything will be out of reach
|
||||
e.cur += maxMatchOffset + int32(len(e.hist))
|
||||
e.hist = e.hist[:0]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,274 @@
|
|||
// +build generate
|
||||
|
||||
//go:generate go run $GOFILE && gofmt -w inflate_gen.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
f, err := os.Create("inflate_gen.go")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer f.Close()
|
||||
types := []string{"*bytes.Buffer", "*bytes.Reader", "*bufio.Reader", "*strings.Reader"}
|
||||
names := []string{"BytesBuffer", "BytesReader", "BufioReader", "StringsReader"}
|
||||
imports := []string{"bytes", "bufio", "io", "strings", "math/bits"}
|
||||
f.WriteString(`// Code generated by go generate gen_inflate.go. DO NOT EDIT.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
`)
|
||||
|
||||
for _, imp := range imports {
|
||||
f.WriteString("\t\"" + imp + "\"\n")
|
||||
}
|
||||
f.WriteString(")\n\n")
|
||||
|
||||
template := `
|
||||
|
||||
// Decode a single Huffman block from f.
|
||||
// hl and hd are the Huffman states for the lit/length values
|
||||
// and the distance values, respectively. If hd == nil, using the
|
||||
// fixed distance encoding associated with fixed Huffman blocks.
|
||||
func (f *decompressor) $FUNCNAME$() {
|
||||
const (
|
||||
stateInit = iota // Zero value must be stateInit
|
||||
stateDict
|
||||
)
|
||||
fr := f.r.($TYPE$)
|
||||
moreBits := func() error {
|
||||
c, err := fr.ReadByte()
|
||||
if err != nil {
|
||||
return noEOF(err)
|
||||
}
|
||||
f.roffset++
|
||||
f.b |= uint32(c) << f.nb
|
||||
f.nb += 8
|
||||
return nil
|
||||
}
|
||||
|
||||
switch f.stepState {
|
||||
case stateInit:
|
||||
goto readLiteral
|
||||
case stateDict:
|
||||
goto copyHistory
|
||||
}
|
||||
|
||||
readLiteral:
|
||||
// Read literal and/or (length, distance) according to RFC section 3.2.3.
|
||||
{
|
||||
var v int
|
||||
{
|
||||
// Inlined v, err := f.huffSym(f.hl)
|
||||
// Since a huffmanDecoder can be empty or be composed of a degenerate tree
|
||||
// with single element, huffSym must error on these two edge cases. In both
|
||||
// cases, the chunks slice will be 0 for the invalid sequence, leading it
|
||||
// satisfy the n == 0 check below.
|
||||
n := uint(f.hl.maxRead)
|
||||
// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,
|
||||
// but is smart enough to keep local variables in registers, so use nb and b,
|
||||
// inline call to moreBits and reassign b,nb back to f on return.
|
||||
nb, b := f.nb, f.b
|
||||
for {
|
||||
for nb < n {
|
||||
c, err := fr.ReadByte()
|
||||
if err != nil {
|
||||
f.b = b
|
||||
f.nb = nb
|
||||
f.err = noEOF(err)
|
||||
return
|
||||
}
|
||||
f.roffset++
|
||||
b |= uint32(c) << (nb & 31)
|
||||
nb += 8
|
||||
}
|
||||
chunk := f.hl.chunks[b&(huffmanNumChunks-1)]
|
||||
n = uint(chunk & huffmanCountMask)
|
||||
if n > huffmanChunkBits {
|
||||
chunk = f.hl.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&f.hl.linkMask]
|
||||
n = uint(chunk & huffmanCountMask)
|
||||
}
|
||||
if n <= nb {
|
||||
if n == 0 {
|
||||
f.b = b
|
||||
f.nb = nb
|
||||
if debugDecode {
|
||||
fmt.Println("huffsym: n==0")
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
return
|
||||
}
|
||||
f.b = b >> (n & 31)
|
||||
f.nb = nb - n
|
||||
v = int(chunk >> huffmanValueShift)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var n uint // number of bits extra
|
||||
var length int
|
||||
var err error
|
||||
switch {
|
||||
case v < 256:
|
||||
f.dict.writeByte(byte(v))
|
||||
if f.dict.availWrite() == 0 {
|
||||
f.toRead = f.dict.readFlush()
|
||||
f.step = (*decompressor).$FUNCNAME$
|
||||
f.stepState = stateInit
|
||||
return
|
||||
}
|
||||
goto readLiteral
|
||||
case v == 256:
|
||||
f.finishBlock()
|
||||
return
|
||||
// otherwise, reference to older data
|
||||
case v < 265:
|
||||
length = v - (257 - 3)
|
||||
n = 0
|
||||
case v < 269:
|
||||
length = v*2 - (265*2 - 11)
|
||||
n = 1
|
||||
case v < 273:
|
||||
length = v*4 - (269*4 - 19)
|
||||
n = 2
|
||||
case v < 277:
|
||||
length = v*8 - (273*8 - 35)
|
||||
n = 3
|
||||
case v < 281:
|
||||
length = v*16 - (277*16 - 67)
|
||||
n = 4
|
||||
case v < 285:
|
||||
length = v*32 - (281*32 - 131)
|
||||
n = 5
|
||||
case v < maxNumLit:
|
||||
length = 258
|
||||
n = 0
|
||||
default:
|
||||
if debugDecode {
|
||||
fmt.Println(v, ">= maxNumLit")
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
for f.nb < n {
|
||||
if err = moreBits(); err != nil {
|
||||
if debugDecode {
|
||||
fmt.Println("morebits n>0:", err)
|
||||
}
|
||||
f.err = err
|
||||
return
|
||||
}
|
||||
}
|
||||
length += int(f.b & uint32(1<<n-1))
|
||||
f.b >>= n
|
||||
f.nb -= n
|
||||
}
|
||||
|
||||
var dist int
|
||||
if f.hd == nil {
|
||||
for f.nb < 5 {
|
||||
if err = moreBits(); err != nil {
|
||||
if debugDecode {
|
||||
fmt.Println("morebits f.nb<5:", err)
|
||||
}
|
||||
f.err = err
|
||||
return
|
||||
}
|
||||
}
|
||||
dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3)))
|
||||
f.b >>= 5
|
||||
f.nb -= 5
|
||||
} else {
|
||||
if dist, err = f.huffSym(f.hd); err != nil {
|
||||
if debugDecode {
|
||||
fmt.Println("huffsym:", err)
|
||||
}
|
||||
f.err = err
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case dist < 4:
|
||||
dist++
|
||||
case dist < maxNumDist:
|
||||
nb := uint(dist-2) >> 1
|
||||
// have 1 bit in bottom of dist, need nb more.
|
||||
extra := (dist & 1) << nb
|
||||
for f.nb < nb {
|
||||
if err = moreBits(); err != nil {
|
||||
if debugDecode {
|
||||
fmt.Println("morebits f.nb<nb:", err)
|
||||
}
|
||||
f.err = err
|
||||
return
|
||||
}
|
||||
}
|
||||
extra |= int(f.b & uint32(1<<nb-1))
|
||||
f.b >>= nb
|
||||
f.nb -= nb
|
||||
dist = 1<<(nb+1) + 1 + extra
|
||||
default:
|
||||
if debugDecode {
|
||||
fmt.Println("dist too big:", dist, maxNumDist)
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
return
|
||||
}
|
||||
|
||||
// No check on length; encoding can be prescient.
|
||||
if dist > f.dict.histSize() {
|
||||
if debugDecode {
|
||||
fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize())
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
return
|
||||
}
|
||||
|
||||
f.copyLen, f.copyDist = length, dist
|
||||
goto copyHistory
|
||||
}
|
||||
|
||||
copyHistory:
|
||||
// Perform a backwards copy according to RFC section 3.2.3.
|
||||
{
|
||||
cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen)
|
||||
if cnt == 0 {
|
||||
cnt = f.dict.writeCopy(f.copyDist, f.copyLen)
|
||||
}
|
||||
f.copyLen -= cnt
|
||||
|
||||
if f.dict.availWrite() == 0 || f.copyLen > 0 {
|
||||
f.toRead = f.dict.readFlush()
|
||||
f.step = (*decompressor).$FUNCNAME$ // We need to continue this work
|
||||
f.stepState = stateDict
|
||||
return
|
||||
}
|
||||
goto readLiteral
|
||||
}
|
||||
}
|
||||
|
||||
`
|
||||
for i, t := range types {
|
||||
s := strings.Replace(template, "$FUNCNAME$", "huffman"+names[i], -1)
|
||||
s = strings.Replace(s, "$TYPE$", t, -1)
|
||||
f.WriteString(s)
|
||||
}
|
||||
f.WriteString("func (f *decompressor) huffmanBlockDecoder() func() {\n")
|
||||
f.WriteString("\tswitch f.r.(type) {\n")
|
||||
for i, t := range types {
|
||||
f.WriteString("\t\tcase " + t + ":\n")
|
||||
f.WriteString("\t\t\treturn f.huffman" + names[i] + "\n")
|
||||
}
|
||||
f.WriteString("\t\tdefault:\n")
|
||||
f.WriteString("\t\t\treturn f.huffmanBlockGeneric")
|
||||
f.WriteString("\t}\n}\n")
|
||||
}
|
||||
|
|
@ -93,12 +93,12 @@ type huffmanBitWriter struct {
|
|||
err error
|
||||
lastHeader int
|
||||
// Set between 0 (reused block can be up to 2x the size)
|
||||
logReusePenalty uint
|
||||
lastHuffMan bool
|
||||
bytes [256]byte
|
||||
literalFreq [lengthCodesStart + 32]uint16
|
||||
offsetFreq [32]uint16
|
||||
codegenFreq [codegenCodeCount]uint16
|
||||
logNewTablePenalty uint
|
||||
lastHuffMan bool
|
||||
bytes [256]byte
|
||||
literalFreq [lengthCodesStart + 32]uint16
|
||||
offsetFreq [32]uint16
|
||||
codegenFreq [codegenCodeCount]uint16
|
||||
|
||||
// codegen must have an extra space for the final symbol.
|
||||
codegen [literalCount + offsetCodeCount + 1]uint8
|
||||
|
|
@ -119,7 +119,7 @@ type huffmanBitWriter struct {
|
|||
// If lastHuffMan is set, a table for outputting literals has been generated and offsets are invalid.
|
||||
//
|
||||
// An incoming block estimates the output size of a new table using a 'fresh' by calculating the
|
||||
// optimal size and adding a penalty in 'logReusePenalty'.
|
||||
// optimal size and adding a penalty in 'logNewTablePenalty'.
|
||||
// A Huffman table is not optimal, which is why we add a penalty, and generating a new table
|
||||
// is slower both for compression and decompression.
|
||||
|
||||
|
|
@ -135,7 +135,6 @@ func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
|
|||
func (w *huffmanBitWriter) reset(writer io.Writer) {
|
||||
w.writer = writer
|
||||
w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
|
||||
w.bytes = [256]byte{}
|
||||
w.lastHeader = 0
|
||||
w.lastHuffMan = false
|
||||
}
|
||||
|
|
@ -178,6 +177,11 @@ func (w *huffmanBitWriter) flush() {
|
|||
w.nbits = 0
|
||||
return
|
||||
}
|
||||
if w.lastHeader > 0 {
|
||||
// We owe an EOB
|
||||
w.writeCode(w.literalEncoding.codes[endBlockMarker])
|
||||
w.lastHeader = 0
|
||||
}
|
||||
n := w.nbytes
|
||||
for w.nbits != 0 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
|
|
@ -350,6 +354,13 @@ func (w *huffmanBitWriter) headerSize() (size, numCodegens int) {
|
|||
int(w.codegenFreq[18])*7, numCodegens
|
||||
}
|
||||
|
||||
// dynamicSize returns the size of dynamically encoded data in bits.
|
||||
func (w *huffmanBitWriter) dynamicReuseSize(litEnc, offEnc *huffmanEncoder) (size int) {
|
||||
size = litEnc.bitLength(w.literalFreq[:]) +
|
||||
offEnc.bitLength(w.offsetFreq[:])
|
||||
return size
|
||||
}
|
||||
|
||||
// dynamicSize returns the size of dynamically encoded data in bits.
|
||||
func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
|
||||
header, numCodegens := w.headerSize()
|
||||
|
|
@ -452,30 +463,30 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n
|
|||
|
||||
i := 0
|
||||
for {
|
||||
var codeWord int = int(w.codegen[i])
|
||||
var codeWord = uint32(w.codegen[i])
|
||||
i++
|
||||
if codeWord == badCode {
|
||||
break
|
||||
}
|
||||
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
|
||||
w.writeCode(w.codegenEncoding.codes[codeWord])
|
||||
|
||||
switch codeWord {
|
||||
case 16:
|
||||
w.writeBits(int32(w.codegen[i]), 2)
|
||||
i++
|
||||
break
|
||||
case 17:
|
||||
w.writeBits(int32(w.codegen[i]), 3)
|
||||
i++
|
||||
break
|
||||
case 18:
|
||||
w.writeBits(int32(w.codegen[i]), 7)
|
||||
i++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeStoredHeader will write a stored header.
|
||||
// If the stored block is only used for EOF,
|
||||
// it is replaced with a fixed huffman block.
|
||||
func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
|
||||
if w.err != nil {
|
||||
return
|
||||
|
|
@ -485,6 +496,16 @@ func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
|
|||
w.writeCode(w.literalEncoding.codes[endBlockMarker])
|
||||
w.lastHeader = 0
|
||||
}
|
||||
|
||||
// To write EOF, use a fixed encoding block. 10 bits instead of 5 bytes.
|
||||
if length == 0 && isEof {
|
||||
w.writeFixedHeader(isEof)
|
||||
// EOB: 7 bits, value: 0
|
||||
w.writeBits(0, 7)
|
||||
w.flush()
|
||||
return
|
||||
}
|
||||
|
||||
var flag int32
|
||||
if isEof {
|
||||
flag = 1
|
||||
|
|
@ -591,8 +612,8 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
|
|||
tokens.AddEOB()
|
||||
}
|
||||
|
||||
// We cannot reuse pure huffman table.
|
||||
if w.lastHuffMan && w.lastHeader > 0 {
|
||||
// We cannot reuse pure huffman table, and must mark as EOF.
|
||||
if (w.lastHuffMan || eof) && w.lastHeader > 0 {
|
||||
// We will not try to reuse.
|
||||
w.writeCode(w.literalEncoding.codes[endBlockMarker])
|
||||
w.lastHeader = 0
|
||||
|
|
@ -606,14 +627,14 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
|
|||
var size int
|
||||
// Check if we should reuse.
|
||||
if w.lastHeader > 0 {
|
||||
// Estimate size for using a new table
|
||||
// Estimate size for using a new table.
|
||||
// Use the previous header size as the best estimate.
|
||||
newSize := w.lastHeader + tokens.EstimatedBits()
|
||||
newSize += newSize >> w.logNewTablePenalty
|
||||
|
||||
// The estimated size is calculated as an optimal table.
|
||||
// We add a penalty to make it more realistic and re-use a bit more.
|
||||
newSize += newSize >> (w.logReusePenalty & 31)
|
||||
extra := w.extraBitSize()
|
||||
reuseSize, _ := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extra)
|
||||
reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + w.extraBitSize()
|
||||
|
||||
// Check if a new table is better.
|
||||
if newSize < reuseSize {
|
||||
|
|
@ -805,21 +826,30 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
|
|||
}
|
||||
|
||||
// Add everything as literals
|
||||
estBits := histogramSize(input, w.literalFreq[:], !eof && !sync) + 15
|
||||
// We have to estimate the header size.
|
||||
// Assume header is around 70 bytes:
|
||||
// https://stackoverflow.com/a/25454430
|
||||
const guessHeaderSizeBits = 70 * 8
|
||||
estBits, estExtra := histogramSize(input, w.literalFreq[:], !eof && !sync)
|
||||
estBits += w.lastHeader + 15
|
||||
if w.lastHeader == 0 {
|
||||
estBits += guessHeaderSizeBits
|
||||
}
|
||||
estBits += estBits >> w.logNewTablePenalty
|
||||
|
||||
// Store bytes, if we don't get a reasonable improvement.
|
||||
ssize, storable := w.storedSize(input)
|
||||
if storable && ssize < (estBits+estBits>>4) {
|
||||
if storable && ssize < estBits {
|
||||
w.writeStoredHeader(len(input), eof)
|
||||
w.writeBytes(input)
|
||||
return
|
||||
}
|
||||
|
||||
if w.lastHeader > 0 {
|
||||
size, _ := w.dynamicSize(w.literalEncoding, huffOffset, w.lastHeader)
|
||||
estBits += estBits >> (w.logReusePenalty)
|
||||
reuseSize := w.literalEncoding.bitLength(w.literalFreq[:256])
|
||||
estBits += estExtra
|
||||
|
||||
if estBits < size {
|
||||
if estBits < reuseSize {
|
||||
// We owe an EOB
|
||||
w.writeCode(w.literalEncoding.codes[endBlockMarker])
|
||||
w.lastHeader = 0
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ package flate
|
|||
import (
|
||||
"math"
|
||||
"math/bits"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -25,8 +24,6 @@ type huffmanEncoder struct {
|
|||
codes []hcode
|
||||
freqcache []literalNode
|
||||
bitCount [17]int32
|
||||
lns byLiteral // stored to avoid repeated allocation in generate
|
||||
lfs byFreq // stored to avoid repeated allocation in generate
|
||||
}
|
||||
|
||||
type literalNode struct {
|
||||
|
|
@ -85,17 +82,14 @@ func generateFixedLiteralEncoding() *huffmanEncoder {
|
|||
// size 8, 000110000 .. 10111111
|
||||
bits = ch + 48
|
||||
size = 8
|
||||
break
|
||||
case ch < 256:
|
||||
// size 9, 110010000 .. 111111111
|
||||
bits = ch + 400 - 144
|
||||
size = 9
|
||||
break
|
||||
case ch < 280:
|
||||
// size 7, 0000000 .. 0010111
|
||||
bits = ch - 256
|
||||
size = 7
|
||||
break
|
||||
default:
|
||||
// size 8, 11000000 .. 11000111
|
||||
bits = ch + 192 - 280
|
||||
|
|
@ -115,8 +109,8 @@ func generateFixedOffsetEncoding() *huffmanEncoder {
|
|||
return h
|
||||
}
|
||||
|
||||
var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
|
||||
var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
|
||||
var fixedLiteralEncoding = generateFixedLiteralEncoding()
|
||||
var fixedOffsetEncoding = generateFixedOffsetEncoding()
|
||||
|
||||
func (h *huffmanEncoder) bitLength(freq []uint16) int {
|
||||
var total int
|
||||
|
|
@ -273,7 +267,7 @@ func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalN
|
|||
// assigned in literal order (not frequency order).
|
||||
chunk := list[len(list)-int(bits):]
|
||||
|
||||
h.lns.sort(chunk)
|
||||
sortByLiteral(chunk)
|
||||
for _, node := range chunk {
|
||||
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
|
||||
code++
|
||||
|
|
@ -318,7 +312,7 @@ func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
|
|||
}
|
||||
return
|
||||
}
|
||||
h.lfs.sort(list)
|
||||
sortByFreq(list)
|
||||
|
||||
// Get the number of literals for each bit count
|
||||
bitCount := h.bitCounts(list, maxBits)
|
||||
|
|
@ -326,59 +320,44 @@ func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
|
|||
h.assignEncodingAndSize(bitCount, list)
|
||||
}
|
||||
|
||||
type byLiteral []literalNode
|
||||
|
||||
func (s *byLiteral) sort(a []literalNode) {
|
||||
*s = byLiteral(a)
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func (s byLiteral) Len() int { return len(s) }
|
||||
|
||||
func (s byLiteral) Less(i, j int) bool {
|
||||
return s[i].literal < s[j].literal
|
||||
}
|
||||
|
||||
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
type byFreq []literalNode
|
||||
|
||||
func (s *byFreq) sort(a []literalNode) {
|
||||
*s = byFreq(a)
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func (s byFreq) Len() int { return len(s) }
|
||||
|
||||
func (s byFreq) Less(i, j int) bool {
|
||||
if s[i].freq == s[j].freq {
|
||||
return s[i].literal < s[j].literal
|
||||
func atLeastOne(v float32) float32 {
|
||||
if v < 1 {
|
||||
return 1
|
||||
}
|
||||
return s[i].freq < s[j].freq
|
||||
return v
|
||||
}
|
||||
|
||||
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
// histogramSize accumulates a histogram of b in h.
|
||||
// An estimated size in bits is returned.
|
||||
// Unassigned values are assigned '1' in the histogram.
|
||||
// len(h) must be >= 256, and h's elements must be all zeroes.
|
||||
func histogramSize(b []byte, h []uint16, fill bool) int {
|
||||
func histogramSize(b []byte, h []uint16, fill bool) (int, int) {
|
||||
h = h[:256]
|
||||
for _, t := range b {
|
||||
h[t]++
|
||||
}
|
||||
invTotal := 1.0 / float64(len(b))
|
||||
shannon := 0.0
|
||||
single := math.Ceil(-math.Log2(invTotal))
|
||||
for i, v := range h[:] {
|
||||
if v > 0 {
|
||||
n := float64(v)
|
||||
shannon += math.Ceil(-math.Log2(n*invTotal) * n)
|
||||
} else if fill {
|
||||
shannon += single
|
||||
h[i] = 1
|
||||
invTotal := 1.0 / float32(len(b))
|
||||
shannon := float32(0.0)
|
||||
var extra float32
|
||||
if fill {
|
||||
oneBits := atLeastOne(-mFastLog2(invTotal))
|
||||
for i, v := range h[:] {
|
||||
if v > 0 {
|
||||
n := float32(v)
|
||||
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
|
||||
} else {
|
||||
h[i] = 1
|
||||
extra += oneBits
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, v := range h[:] {
|
||||
if v > 0 {
|
||||
n := float32(v)
|
||||
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
|
||||
}
|
||||
}
|
||||
}
|
||||
return int(shannon + 0.99)
|
||||
|
||||
return int(shannon + 0.99), int(extra + 0.99)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
// Sort sorts data.
|
||||
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
|
||||
// data.Less and data.Swap. The sort is not guaranteed to be stable.
|
||||
func sortByFreq(data []literalNode) {
|
||||
n := len(data)
|
||||
quickSortByFreq(data, 0, n, maxDepth(n))
|
||||
}
|
||||
|
||||
func quickSortByFreq(data []literalNode, a, b, maxDepth int) {
|
||||
for b-a > 12 { // Use ShellSort for slices <= 12 elements
|
||||
if maxDepth == 0 {
|
||||
heapSort(data, a, b)
|
||||
return
|
||||
}
|
||||
maxDepth--
|
||||
mlo, mhi := doPivotByFreq(data, a, b)
|
||||
// Avoiding recursion on the larger subproblem guarantees
|
||||
// a stack depth of at most lg(b-a).
|
||||
if mlo-a < b-mhi {
|
||||
quickSortByFreq(data, a, mlo, maxDepth)
|
||||
a = mhi // i.e., quickSortByFreq(data, mhi, b)
|
||||
} else {
|
||||
quickSortByFreq(data, mhi, b, maxDepth)
|
||||
b = mlo // i.e., quickSortByFreq(data, a, mlo)
|
||||
}
|
||||
}
|
||||
if b-a > 1 {
|
||||
// Do ShellSort pass with gap 6
|
||||
// It could be written in this simplified form cause b-a <= 12
|
||||
for i := a + 6; i < b; i++ {
|
||||
if data[i].freq == data[i-6].freq && data[i].literal < data[i-6].literal || data[i].freq < data[i-6].freq {
|
||||
data[i], data[i-6] = data[i-6], data[i]
|
||||
}
|
||||
}
|
||||
insertionSortByFreq(data, a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// siftDownByFreq implements the heap property on data[lo, hi).
|
||||
// first is an offset into the array where the root of the heap lies.
|
||||
func siftDownByFreq(data []literalNode, lo, hi, first int) {
|
||||
root := lo
|
||||
for {
|
||||
child := 2*root + 1
|
||||
if child >= hi {
|
||||
break
|
||||
}
|
||||
if child+1 < hi && (data[first+child].freq == data[first+child+1].freq && data[first+child].literal < data[first+child+1].literal || data[first+child].freq < data[first+child+1].freq) {
|
||||
child++
|
||||
}
|
||||
if data[first+root].freq == data[first+child].freq && data[first+root].literal > data[first+child].literal || data[first+root].freq > data[first+child].freq {
|
||||
return
|
||||
}
|
||||
data[first+root], data[first+child] = data[first+child], data[first+root]
|
||||
root = child
|
||||
}
|
||||
}
|
||||
func doPivotByFreq(data []literalNode, lo, hi int) (midlo, midhi int) {
|
||||
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
|
||||
if hi-lo > 40 {
|
||||
// Tukey's ``Ninther,'' median of three medians of three.
|
||||
s := (hi - lo) / 8
|
||||
medianOfThreeSortByFreq(data, lo, lo+s, lo+2*s)
|
||||
medianOfThreeSortByFreq(data, m, m-s, m+s)
|
||||
medianOfThreeSortByFreq(data, hi-1, hi-1-s, hi-1-2*s)
|
||||
}
|
||||
medianOfThreeSortByFreq(data, lo, m, hi-1)
|
||||
|
||||
// Invariants are:
|
||||
// data[lo] = pivot (set up by ChoosePivot)
|
||||
// data[lo < i < a] < pivot
|
||||
// data[a <= i < b] <= pivot
|
||||
// data[b <= i < c] unexamined
|
||||
// data[c <= i < hi-1] > pivot
|
||||
// data[hi-1] >= pivot
|
||||
pivot := lo
|
||||
a, c := lo+1, hi-1
|
||||
|
||||
for ; a < c && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ {
|
||||
}
|
||||
b := a
|
||||
for {
|
||||
for ; b < c && (data[pivot].freq == data[b].freq && data[pivot].literal > data[b].literal || data[pivot].freq > data[b].freq); b++ { // data[b] <= pivot
|
||||
}
|
||||
for ; b < c && (data[pivot].freq == data[c-1].freq && data[pivot].literal < data[c-1].literal || data[pivot].freq < data[c-1].freq); c-- { // data[c-1] > pivot
|
||||
}
|
||||
if b >= c {
|
||||
break
|
||||
}
|
||||
// data[b] > pivot; data[c-1] <= pivot
|
||||
data[b], data[c-1] = data[c-1], data[b]
|
||||
b++
|
||||
c--
|
||||
}
|
||||
// If hi-c<3 then there are duplicates (by property of median of nine).
|
||||
// Let's be a bit more conservative, and set border to 5.
|
||||
protect := hi-c < 5
|
||||
if !protect && hi-c < (hi-lo)/4 {
|
||||
// Lets test some points for equality to pivot
|
||||
dups := 0
|
||||
if data[pivot].freq == data[hi-1].freq && data[pivot].literal > data[hi-1].literal || data[pivot].freq > data[hi-1].freq { // data[hi-1] = pivot
|
||||
data[c], data[hi-1] = data[hi-1], data[c]
|
||||
c++
|
||||
dups++
|
||||
}
|
||||
if data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq { // data[b-1] = pivot
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// m-lo = (hi-lo)/2 > 6
|
||||
// b-lo > (hi-lo)*3/4-1 > 8
|
||||
// ==> m < b ==> data[m] <= pivot
|
||||
if data[m].freq == data[pivot].freq && data[m].literal > data[pivot].literal || data[m].freq > data[pivot].freq { // data[m] = pivot
|
||||
data[m], data[b-1] = data[b-1], data[m]
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// if at least 2 points are equal to pivot, assume skewed distribution
|
||||
protect = dups > 1
|
||||
}
|
||||
if protect {
|
||||
// Protect against a lot of duplicates
|
||||
// Add invariant:
|
||||
// data[a <= i < b] unexamined
|
||||
// data[b <= i < c] = pivot
|
||||
for {
|
||||
for ; a < b && (data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq); b-- { // data[b] == pivot
|
||||
}
|
||||
for ; a < b && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { // data[a] < pivot
|
||||
}
|
||||
if a >= b {
|
||||
break
|
||||
}
|
||||
// data[a] == pivot; data[b-1] < pivot
|
||||
data[a], data[b-1] = data[b-1], data[a]
|
||||
a++
|
||||
b--
|
||||
}
|
||||
}
|
||||
// Swap pivot into middle
|
||||
data[pivot], data[b-1] = data[b-1], data[pivot]
|
||||
return b - 1, c
|
||||
}
|
||||
|
||||
// Insertion sort
|
||||
func insertionSortByFreq(data []literalNode, a, b int) {
|
||||
for i := a + 1; i < b; i++ {
|
||||
for j := i; j > a && (data[j].freq == data[j-1].freq && data[j].literal < data[j-1].literal || data[j].freq < data[j-1].freq); j-- {
|
||||
data[j], data[j-1] = data[j-1], data[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// quickSortByFreq, loosely following Bentley and McIlroy,
|
||||
// ``Engineering a Sort Function,'' SP&E November 1993.
|
||||
|
||||
// medianOfThreeSortByFreq moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
|
||||
func medianOfThreeSortByFreq(data []literalNode, m1, m0, m2 int) {
|
||||
// sort 3 elements
|
||||
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
|
||||
data[m1], data[m0] = data[m0], data[m1]
|
||||
}
|
||||
// data[m0] <= data[m1]
|
||||
if data[m2].freq == data[m1].freq && data[m2].literal < data[m1].literal || data[m2].freq < data[m1].freq {
|
||||
data[m2], data[m1] = data[m1], data[m2]
|
||||
// data[m0] <= data[m2] && data[m1] < data[m2]
|
||||
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
|
||||
data[m1], data[m0] = data[m0], data[m1]
|
||||
}
|
||||
}
|
||||
// now data[m0] <= data[m1] <= data[m2]
|
||||
}
|
||||
201
vendor/github.com/klauspost/compress/flate/huffman_sortByLiteral.go
generated
vendored
Normal file
201
vendor/github.com/klauspost/compress/flate/huffman_sortByLiteral.go
generated
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
// Sort sorts data.
|
||||
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
|
||||
// data.Less and data.Swap. The sort is not guaranteed to be stable.
|
||||
func sortByLiteral(data []literalNode) {
|
||||
n := len(data)
|
||||
quickSort(data, 0, n, maxDepth(n))
|
||||
}
|
||||
|
||||
func quickSort(data []literalNode, a, b, maxDepth int) {
|
||||
for b-a > 12 { // Use ShellSort for slices <= 12 elements
|
||||
if maxDepth == 0 {
|
||||
heapSort(data, a, b)
|
||||
return
|
||||
}
|
||||
maxDepth--
|
||||
mlo, mhi := doPivot(data, a, b)
|
||||
// Avoiding recursion on the larger subproblem guarantees
|
||||
// a stack depth of at most lg(b-a).
|
||||
if mlo-a < b-mhi {
|
||||
quickSort(data, a, mlo, maxDepth)
|
||||
a = mhi // i.e., quickSort(data, mhi, b)
|
||||
} else {
|
||||
quickSort(data, mhi, b, maxDepth)
|
||||
b = mlo // i.e., quickSort(data, a, mlo)
|
||||
}
|
||||
}
|
||||
if b-a > 1 {
|
||||
// Do ShellSort pass with gap 6
|
||||
// It could be written in this simplified form cause b-a <= 12
|
||||
for i := a + 6; i < b; i++ {
|
||||
if data[i].literal < data[i-6].literal {
|
||||
data[i], data[i-6] = data[i-6], data[i]
|
||||
}
|
||||
}
|
||||
insertionSort(data, a, b)
|
||||
}
|
||||
}
|
||||
func heapSort(data []literalNode, a, b int) {
|
||||
first := a
|
||||
lo := 0
|
||||
hi := b - a
|
||||
|
||||
// Build heap with greatest element at top.
|
||||
for i := (hi - 1) / 2; i >= 0; i-- {
|
||||
siftDown(data, i, hi, first)
|
||||
}
|
||||
|
||||
// Pop elements, largest first, into end of data.
|
||||
for i := hi - 1; i >= 0; i-- {
|
||||
data[first], data[first+i] = data[first+i], data[first]
|
||||
siftDown(data, lo, i, first)
|
||||
}
|
||||
}
|
||||
|
||||
// siftDown implements the heap property on data[lo, hi).
|
||||
// first is an offset into the array where the root of the heap lies.
|
||||
func siftDown(data []literalNode, lo, hi, first int) {
|
||||
root := lo
|
||||
for {
|
||||
child := 2*root + 1
|
||||
if child >= hi {
|
||||
break
|
||||
}
|
||||
if child+1 < hi && data[first+child].literal < data[first+child+1].literal {
|
||||
child++
|
||||
}
|
||||
if data[first+root].literal > data[first+child].literal {
|
||||
return
|
||||
}
|
||||
data[first+root], data[first+child] = data[first+child], data[first+root]
|
||||
root = child
|
||||
}
|
||||
}
|
||||
func doPivot(data []literalNode, lo, hi int) (midlo, midhi int) {
|
||||
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
|
||||
if hi-lo > 40 {
|
||||
// Tukey's ``Ninther,'' median of three medians of three.
|
||||
s := (hi - lo) / 8
|
||||
medianOfThree(data, lo, lo+s, lo+2*s)
|
||||
medianOfThree(data, m, m-s, m+s)
|
||||
medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
|
||||
}
|
||||
medianOfThree(data, lo, m, hi-1)
|
||||
|
||||
// Invariants are:
|
||||
// data[lo] = pivot (set up by ChoosePivot)
|
||||
// data[lo < i < a] < pivot
|
||||
// data[a <= i < b] <= pivot
|
||||
// data[b <= i < c] unexamined
|
||||
// data[c <= i < hi-1] > pivot
|
||||
// data[hi-1] >= pivot
|
||||
pivot := lo
|
||||
a, c := lo+1, hi-1
|
||||
|
||||
for ; a < c && data[a].literal < data[pivot].literal; a++ {
|
||||
}
|
||||
b := a
|
||||
for {
|
||||
for ; b < c && data[pivot].literal > data[b].literal; b++ { // data[b] <= pivot
|
||||
}
|
||||
for ; b < c && data[pivot].literal < data[c-1].literal; c-- { // data[c-1] > pivot
|
||||
}
|
||||
if b >= c {
|
||||
break
|
||||
}
|
||||
// data[b] > pivot; data[c-1] <= pivot
|
||||
data[b], data[c-1] = data[c-1], data[b]
|
||||
b++
|
||||
c--
|
||||
}
|
||||
// If hi-c<3 then there are duplicates (by property of median of nine).
|
||||
// Let's be a bit more conservative, and set border to 5.
|
||||
protect := hi-c < 5
|
||||
if !protect && hi-c < (hi-lo)/4 {
|
||||
// Lets test some points for equality to pivot
|
||||
dups := 0
|
||||
if data[pivot].literal > data[hi-1].literal { // data[hi-1] = pivot
|
||||
data[c], data[hi-1] = data[hi-1], data[c]
|
||||
c++
|
||||
dups++
|
||||
}
|
||||
if data[b-1].literal > data[pivot].literal { // data[b-1] = pivot
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// m-lo = (hi-lo)/2 > 6
|
||||
// b-lo > (hi-lo)*3/4-1 > 8
|
||||
// ==> m < b ==> data[m] <= pivot
|
||||
if data[m].literal > data[pivot].literal { // data[m] = pivot
|
||||
data[m], data[b-1] = data[b-1], data[m]
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// if at least 2 points are equal to pivot, assume skewed distribution
|
||||
protect = dups > 1
|
||||
}
|
||||
if protect {
|
||||
// Protect against a lot of duplicates
|
||||
// Add invariant:
|
||||
// data[a <= i < b] unexamined
|
||||
// data[b <= i < c] = pivot
|
||||
for {
|
||||
for ; a < b && data[b-1].literal > data[pivot].literal; b-- { // data[b] == pivot
|
||||
}
|
||||
for ; a < b && | ||||