mirror of https://github.com/go-gitea/gitea.git
Issue indexer queue redis support (#6218)
* add redis queue * finished indexer redis queue * add redis vendor * fix vet * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md Co-Authored-By: lunny <xiaolunwen@gmail.com> * switch to go mod * Update required changes for new logging func signaturespull/6546/head
Lunny Xiao
4 years ago
committed by
Lauris BH
parent
6e4af4985e
commit
e7d7dcb090
@ -0,0 +1,145 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package issues |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"errors" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"github.com/go-redis/redis" |
||||
) |
||||
|
||||
var ( |
||||
_ Queue = &RedisQueue{} |
||||
) |
||||
|
||||
type redisClient interface { |
||||
RPush(key string, args ...interface{}) *redis.IntCmd |
||||
LPop(key string) *redis.StringCmd |
||||
Ping() *redis.StatusCmd |
||||
} |
||||
|
||||
// RedisQueue redis queue
|
||||
type RedisQueue struct { |
||||
client redisClient |
||||
queueName string |
||||
indexer Indexer |
||||
batchNumber int |
||||
} |
||||
|
||||
func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { |
||||
fields := strings.Fields(connStr) |
||||
for _, f := range fields { |
||||
items := strings.SplitN(f, "=", 2) |
||||
if len(items) < 2 { |
||||
continue |
||||
} |
||||
switch strings.ToLower(items[0]) { |
||||
case "addrs": |
||||
addrs = items[1] |
||||
case "password": |
||||
password = items[1] |
||||
case "db": |
||||
dbIdx, err = strconv.Atoi(items[1]) |
||||
if err != nil { |
||||
return |
||||
} |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// NewRedisQueue creates single redis or cluster redis queue
|
||||
func NewRedisQueue(addrs string, password string, dbIdx int, indexer Indexer, batchNumber int) (*RedisQueue, error) { |
||||
dbs := strings.Split(addrs, ",") |
||||
var queue = RedisQueue{ |
||||
queueName: "issue_indexer_queue", |
||||
indexer: indexer, |
||||
batchNumber: batchNumber, |
||||
} |
||||
if len(dbs) == 0 { |
||||
return nil, errors.New("no redis host found") |
||||
} else if len(dbs) == 1 { |
||||
queue.client = redis.NewClient(&redis.Options{ |
||||
Addr: strings.TrimSpace(dbs[0]), // use default Addr
|
||||
Password: password, // no password set
|
||||
DB: dbIdx, // use default DB
|
||||
}) |
||||
} else { |
||||
queue.client = redis.NewClusterClient(&redis.ClusterOptions{ |
||||
Addrs: dbs, |
||||
}) |
||||
} |
||||
if err := queue.client.Ping().Err(); err != nil { |
||||
return nil, err |
||||
} |
||||
return &queue, nil |
||||
} |
||||
|
||||
// Run runs the redis queue
|
||||
func (r *RedisQueue) Run() error { |
||||
var i int |
||||
var datas = make([]*IndexerData, 0, r.batchNumber) |
||||
for { |
||||
bs, err := r.client.LPop(r.queueName).Bytes() |
||||
if err != nil && err != redis.Nil { |
||||
log.Error("LPop faile: %v", err) |
||||
time.Sleep(time.Millisecond * 100) |
||||
continue |
||||
} |
||||
|
||||
i++ |
||||
if len(datas) > r.batchNumber || (len(datas) > 0 && i > 3) { |
||||
r.indexer.Index(datas) |
||||
datas = make([]*IndexerData, 0, r.batchNumber) |
||||
i = 0 |
||||
} |
||||
|
||||
if len(bs) <= 0 { |
||||
time.Sleep(time.Millisecond * 100) |
||||
continue |
||||
} |
||||
|
||||
var data IndexerData |
||||
err = json.Unmarshal(bs, &data) |
||||
if err != nil { |
||||
log.Error("Unmarshal: %v", err) |
||||
time.Sleep(time.Millisecond * 100) |
||||
continue |
||||
} |
||||
|
||||
log.Trace("RedisQueue: task found: %#v", data) |
||||
|
||||
if data.IsDelete { |
||||
if data.ID > 0 { |
||||
if err = r.indexer.Delete(data.ID); err != nil { |
||||
log.Error("indexer.Delete: %v", err) |
||||
} |
||||
} else if len(data.IDs) > 0 { |
||||
if err = r.indexer.Delete(data.IDs...); err != nil { |
||||
log.Error("indexer.Delete: %v", err) |
||||
} |
||||
} |
||||
time.Sleep(time.Millisecond * 100) |
||||
continue |
||||
} |
||||
|
||||
datas = append(datas, &data) |
||||
time.Sleep(time.Millisecond * 100) |
||||
} |
||||
} |
||||
|
||||
// Push implements Queue
|
||||
func (r *RedisQueue) Push(data *IndexerData) error { |
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return r.client.RPush(r.queueName, bs).Err() |
||||
} |
||||
@ -0,0 +1,2 @@ |
||||
*.rdb |
||||
testdata/*/ |
||||
@ -0,0 +1,19 @@ |
||||
sudo: false |
||||
language: go |
||||
|
||||
services: |
||||
- redis-server |
||||
|
||||
go: |
||||
- 1.9.x |
||||
- 1.10.x |
||||
- 1.11.x |
||||
- tip |
||||
|
||||
matrix: |
||||
allow_failures: |
||||
- go: tip |
||||
|
||||
install: |
||||
- go get github.com/onsi/ginkgo |
||||
- go get github.com/onsi/gomega |
||||
@ -0,0 +1,25 @@ |
||||
# Changelog |
||||
|
||||
## Unreleased |
||||
|
||||
- Cluster and Ring pipelines process commands for each node in its own goroutine. |
||||
|
||||
## 6.14 |
||||
|
||||
- Added Options.MinIdleConns. |
||||
- Added Options.MaxConnAge. |
||||
- PoolStats.FreeConns is renamed to PoolStats.IdleConns. |
||||
- Add Client.Do to simplify creating custom commands. |
||||
- Add Cmd.String, Cmd.Int, Cmd.Int64, Cmd.Uint64, Cmd.Float64, and Cmd.Bool helpers. |
||||
- Lower memory usage. |
||||
|
||||
## v6.13 |
||||
|
||||
- Ring got new options called `HashReplicas` and `Hash`. It is recommended to set `HashReplicas = 1000` for better keys distribution between shards. |
||||
- Cluster client was optimized to use much less memory when reloading cluster state. |
||||
- PubSub.ReceiveMessage is re-worked to not use ReceiveTimeout so it does not lose data when timeout occurres. In most cases it is recommended to use PubSub.Channel instead. |
||||
- Dialer.KeepAlive is set to 5 minutes by default. |
||||
|
||||
## v6.12 |
||||
|
||||
- ClusterClient got new option called `ClusterSlots` which allows to build cluster of normal Redis Servers that don't have cluster mode enabled. See https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup |
||||
@ -0,0 +1,25 @@ |
||||
Copyright (c) 2013 The github.com/go-redis/redis Authors. |
||||
All rights reserved. |
||||
|
||||
Redistribution and use in source and binary forms, with or without |
||||
modification, are permitted provided that the following conditions are |
||||
met: |
||||
|
||||
* Redistributions of source code must retain the above copyright |
||||
notice, this list of conditions and the following disclaimer. |
||||
* Redistributions in binary form must reproduce the above |
||||
copyright notice, this list of conditions and the following disclaimer |
||||
in the documentation and/or other materials provided with the |
||||
distribution. |
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
@ -0,0 +1,22 @@ |
||||
all: testdeps |
||||
go test ./...
|
||||
go test ./... -short -race
|
||||
env GOOS=linux GOARCH=386 go test ./...
|
||||
go vet
|
||||
go get github.com/gordonklaus/ineffassign
|
||||
ineffassign .
|
||||
|
||||
testdeps: testdata/redis/src/redis-server |
||||
|
||||
bench: testdeps |
||||
go test ./... -test.run=NONE -test.bench=. -test.benchmem
|
||||
|
||||
.PHONY: all test testdeps bench |
||||
|
||||
testdata/redis: |
||||
mkdir -p $@
|
||||
wget -qO- https://github.com/antirez/redis/archive/5.0.tar.gz | tar xvz --strip-components=1 -C $@
|
||||
|
||||
testdata/redis/src/redis-server: testdata/redis |
||||
sed -i.bak 's/libjemalloc.a/libjemalloc.a -lrt/g' $</src/Makefile
|
||||
cd $< && make all
|
||||
@ -0,0 +1,146 @@ |
||||
# Redis client for Golang |
||||
|
||||
[](https://travis-ci.org/go-redis/redis) |
||||
[](https://godoc.org/github.com/go-redis/redis) |
||||
[](https://airbrake.io) |
||||
|
||||
Supports: |
||||
|
||||
- Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. |
||||
- Automatic connection pooling with [circuit breaker](https://en.wikipedia.org/wiki/Circuit_breaker_design_pattern) support. |
||||
- [Pub/Sub](https://godoc.org/github.com/go-redis/redis#PubSub). |
||||
- [Transactions](https://godoc.org/github.com/go-redis/redis#Multi). |
||||
- [Pipeline](https://godoc.org/github.com/go-redis/redis#example-Client-Pipeline) and [TxPipeline](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline). |
||||
- [Scripting](https://godoc.org/github.com/go-redis/redis#Script). |
||||
- [Timeouts](https://godoc.org/github.com/go-redis/redis#Options). |
||||
- [Redis Sentinel](https://godoc.org/github.com/go-redis/redis#NewFailoverClient). |
||||
- [Redis Cluster](https://godoc.org/github.com/go-redis/redis#NewClusterClient). |
||||
- [Cluster of Redis Servers](https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup) without using cluster mode and Redis Sentinel. |
||||
- [Ring](https://godoc.org/github.com/go-redis/redis#NewRing). |
||||
- [Instrumentation](https://godoc.org/github.com/go-redis/redis#ex-package--Instrumentation). |
||||
- [Cache friendly](https://github.com/go-redis/cache). |
||||
- [Rate limiting](https://github.com/go-redis/redis_rate). |
||||
- [Distributed Locks](https://github.com/bsm/redis-lock). |
||||
|
||||
API docs: https://godoc.org/github.com/go-redis/redis. |
||||
Examples: https://godoc.org/github.com/go-redis/redis#pkg-examples. |
||||
|
||||
## Installation |
||||
|
||||
Install: |
||||
|
||||
```shell |
||||
go get -u github.com/go-redis/redis |
||||
``` |
||||
|
||||
Import: |
||||
|
||||
```go |
||||
import "github.com/go-redis/redis" |
||||
``` |
||||
|
||||
## Quickstart |
||||
|
||||
```go |
||||
func ExampleNewClient() { |
||||
client := redis.NewClient(&redis.Options{ |
||||
Addr: "localhost:6379", |
||||
Password: "", // no password set |
||||
DB: 0, // use default DB |
||||
}) |
||||
|
||||
pong, err := client.Ping().Result() |
||||
fmt.Println(pong, err) |
||||
// Output: PONG <nil> |
||||
} |
||||
|
||||
func ExampleClient() { |
||||
err := client.Set("key", "value", 0).Err() |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
|
||||
val, err := client.Get("key").Result() |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
fmt.Println("key", val) |
||||
|
||||
val2, err := client.Get("key2").Result() |
||||
if err == redis.Nil { |
||||
fmt.Println("key2 does not exist") |
||||
} else if err != nil { |
||||
panic(err) |
||||
} else { |
||||
fmt.Println("key2", val2) |
||||
} |
||||
// Output: key value |
||||
// key2 does not exist |
||||
} |
||||
``` |
||||
|
||||
## Howto |
||||
|
||||
Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-examples) to get an idea how to use this package. |
||||
|
||||
## Look and feel |
||||
|
||||
Some corner cases: |
||||
|
||||
```go |
||||
// SET key value EX 10 NX |
||||
set, err := client.SetNX("key", "value", 10*time.Second).Result() |
||||
|
||||
// SORT list LIMIT 0 2 ASC |
||||
vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() |
||||
|
||||
// ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2 |
||||
vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ |
||||
Min: "-inf", |
||||
Max: "+inf", |
||||
Offset: 0, |
||||
Count: 2, |
||||
}).Result() |
||||
|
||||
// ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM |
||||
vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result() |
||||
|
||||
// EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" |
||||
vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result() |
||||
``` |
||||
|
||||
## Benchmark |
||||
|
||||
go-redis vs redigo: |
||||
|
||||
``` |
||||
BenchmarkSetGoRedis10Conns64Bytes-4 200000 7621 ns/op 210 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis100Conns64Bytes-4 200000 7554 ns/op 210 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis10Conns1KB-4 200000 7697 ns/op 210 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis100Conns1KB-4 200000 7688 ns/op 210 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis10Conns10KB-4 200000 9214 ns/op 210 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis100Conns10KB-4 200000 9181 ns/op 210 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis10Conns1MB-4 2000 583242 ns/op 2337 B/op 6 allocs/op |
||||
BenchmarkSetGoRedis100Conns1MB-4 2000 583089 ns/op 2338 B/op 6 allocs/op |
||||
BenchmarkSetRedigo10Conns64Bytes-4 200000 7576 ns/op 208 B/op 7 allocs/op |
||||
BenchmarkSetRedigo100Conns64Bytes-4 200000 7782 ns/op 208 B/op 7 allocs/op |
||||
BenchmarkSetRedigo10Conns1KB-4 200000 7958 ns/op 208 B/op 7 allocs/op |
||||
BenchmarkSetRedigo100Conns1KB-4 200000 7725 ns/op 208 B/op 7 allocs/op |
||||
BenchmarkSetRedigo10Conns10KB-4 100000 18442 ns/op 208 B/op 7 allocs/op |
||||
BenchmarkSetRedigo100Conns10KB-4 100000 18818 ns/op 208 B/op 7 allocs/op |
||||
BenchmarkSetRedigo10Conns1MB-4 2000 668829 ns/op 226 B/op 7 allocs/op |
||||
BenchmarkSetRedigo100Conns1MB-4 2000 679542 ns/op 226 B/op 7 allocs/op |
||||
``` |
||||
|
||||
Redis Cluster: |
||||
|
||||
``` |
||||
BenchmarkRedisPing-4 200000 6983 ns/op 116 B/op 4 allocs/op |
||||
BenchmarkRedisClusterPing-4 100000 11535 ns/op 117 B/op 4 allocs/op |
||||
``` |
||||
|
||||
## See also |
||||
|
||||
- [Golang PostgreSQL ORM](https://github.com/go-pg/pg) |
||||
- [Golang msgpack](https://github.com/vmihailenco/msgpack) |
||||
- [Golang message task queue](https://github.com/go-msgqueue/msgqueue) |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,22 @@ |
||||
package redis |
||||
|
||||
import "sync/atomic" |
||||
|
||||
func (c *ClusterClient) DBSize() *IntCmd { |
||||
cmd := NewIntCmd("dbsize") |
||||
var size int64 |
||||
err := c.ForEachMaster(func(master *Client) error { |
||||
n, err := master.DBSize().Result() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
atomic.AddInt64(&size, n) |
||||
return nil |
||||
}) |
||||
if err != nil { |
||||
cmd.setErr(err) |
||||
return cmd |
||||
} |
||||
cmd.val = size |
||||
return cmd |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,4 @@ |
||||
/* |
||||
Package redis implements a Redis client. |
||||
*/ |
||||
package redis |
||||
@ -0,0 +1,81 @@ |
||||
/* |
||||
Copyright 2013 Google Inc. |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
|
||||
// Package consistenthash provides an implementation of a ring hash.
|
||||
package consistenthash |
||||
|
||||
import ( |
||||
"hash/crc32" |
||||
"sort" |
||||
"strconv" |
||||
) |
||||
|
||||
type Hash func(data []byte) uint32 |
||||
|
||||
type Map struct { |
||||
hash Hash |
||||
replicas int |
||||
keys []int // Sorted
|
||||
hashMap map[int]string |
||||
} |
||||
|
||||
func New(replicas int, fn Hash) *Map { |
||||
m := &Map{ |
||||
replicas: replicas, |
||||
hash: fn, |
||||
hashMap: make(map[int]string), |
||||
} |
||||
if m.hash == nil { |
||||
m.hash = crc32.ChecksumIEEE |
||||
} |
||||
return m |
||||
} |
||||
|
||||
// Returns true if there are no items available.
|
||||
func (m *Map) IsEmpty() bool { |
||||
return len(m.keys) == 0 |
||||
} |
||||
|
||||
// Adds some keys to the hash.
|
||||
func (m *Map) Add(keys ...string) { |
||||
for _, key := range keys { |
||||
for i := 0; i < m.replicas; i++ { |
||||
hash := int(m.hash([]byte(strconv.Itoa(i) + key))) |
||||
m.keys = append(m.keys, hash) |
||||
m.hashMap[hash] = key |
||||
} |
||||
} |
||||
sort.Ints(m.keys) |
||||
} |
||||
|
||||
// Gets the closest item in the hash to the provided key.
|
||||
func (m *Map) Get(key string) string { |
||||
if m.IsEmpty() { |
||||
return "" |
||||
} |
||||
|
||||
hash := int(m.hash([]byte(key))) |
||||
|
||||
// Binary search for appropriate replica.
|
||||
idx := sort.Search(len(m.keys), func(i int) bool { return m.keys[i] >= hash }) |
||||
|
||||
// Means we have cycled back to the first replica.
|
||||
if idx == len(m.keys) { |
||||
idx = 0 |
||||
} |
||||
|
||||
return m.hashMap[m.keys[idx]] |
||||
} |
||||
@ -0,0 +1,89 @@ |
||||
package internal |
||||
|
||||
import ( |
||||
"io" |
||||
"net" |
||||
"strings" |
||||
|
||||
"github.com/go-redis/redis/internal/proto" |
||||
) |
||||
|
||||
func IsRetryableError(err error, retryTimeout bool) bool { |
||||
if err == nil { |
||||
return false |
||||
} |
||||
if err == io.EOF { |
||||
return true |
||||
} |
||||
if netErr, ok := err.(net.Error); ok { |
||||
if netErr.Timeout() { |
||||
return retryTimeout |
||||
} |
||||
return true |
||||
} |
||||
s := err.Error() |
||||
if s == "ERR max number of clients reached" { |
||||
return true |
||||
} |
||||
if strings.HasPrefix(s, "LOADING ") { |
||||
return true |
||||
} |
||||
if strings.HasPrefix(s, "READONLY ") { |
||||
return true |
||||
} |
||||
if strings.HasPrefix(s, "CLUSTERDOWN ") { |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func IsRedisError(err error) bool { |
||||
_, ok := err.(proto.RedisError) |
||||
return ok |
||||
} |
||||
|
||||
func IsBadConn(err error, allowTimeout bool) bool { |
||||
if err == nil { |
||||
return false |
||||
} |
||||
if IsRedisError(err) { |
||||
// #790
|
||||
return IsReadOnlyError(err) |
||||
} |
||||
if allowTimeout { |
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() { |
||||
return false |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func IsMovedError(err error) (moved bool, ask bool, addr string) { |
||||
if !IsRedisError(err) { |
||||
return |
||||
} |
||||
|
||||
s := err.Error() |
||||
if strings.HasPrefix(s, "MOVED ") { |
||||
moved = true |
||||
} else if strings.HasPrefix(s, "ASK ") { |
||||
ask = true |
||||
} else { |
||||
return |
||||
} |
||||
|
||||
ind := strings.LastIndex(s, " ") |
||||
if ind == -1 { |
||||
return false, false, "" |
||||
} |
||||
addr = s[ind+1:] |
||||
return |
||||
} |
||||
|
||||
func IsLoadingError(err error) bool { |
||||
return strings.HasPrefix(err.Error(), "LOADING ") |
||||
} |
||||
|
||||
func IsReadOnlyError(err error) bool { |
||||
return strings.HasPrefix(err.Error(), "READONLY ") |
||||
} |
||||
@ -0,0 +1,77 @@ |
||||
package hashtag |
||||
|
||||
import ( |
||||
"math/rand" |
||||
"strings" |
||||
) |
||||
|
||||
const slotNumber = 16384 |
||||
|
||||
// CRC16 implementation according to CCITT standards.
|
||||
// Copyright 2001-2010 Georges Menie (www.menie.org)
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// http://redis.io/topics/cluster-spec#appendix-a-crc16-reference-implementation-in-ansi-c
|
||||
var crc16tab = [256]uint16{ |
||||
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, |
||||
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, |
||||
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, |
||||
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, |
||||
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, |
||||
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, |
||||
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, |
||||
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, |
||||
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, |
||||
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, |
||||
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, |
||||
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, |
||||
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, |
||||
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, |
||||
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, |
||||
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, |
||||
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, |
||||
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, |
||||
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, |
||||
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, |
||||
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, |
||||
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, |
||||
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, |
||||
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, |
||||
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, |
||||
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, |
||||
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, |
||||
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, |
||||
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, |
||||
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, |
||||
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, |
||||
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, |
||||
} |
||||
|
||||
func Key(key string) string { |
||||
if s := strings.IndexByte(key, '{'); s > -1 { |
||||
if e := strings.IndexByte(key[s+1:], '}'); e > 0 { |
||||
return key[s+1 : s+e+1] |
||||
} |
||||
} |
||||
return key |
||||
} |
||||
|
||||
func RandomSlot() int { |
||||
return rand.Intn(slotNumber) |
||||
} |
||||
|
||||
// hashSlot returns a consistent slot number between 0 and 16383
|
||||
// for any given string key.
|
||||
func Slot(key string) int { |
||||
if key == "" { |
||||
return RandomSlot() |
||||
} |
||||
key = Key(key) |
||||
return int(crc16sum(key)) % slotNumber |
||||
} |
||||
|
||||
func crc16sum(key string) (crc uint16) { |
||||
for i := 0; i < len(key); i++ { |
||||
crc = (crc << 8) ^ crc16tab[(byte(crc>>8)^key[i])&0x00ff] |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
package internal |
||||
|
||||
import ( |
||||
"math/rand" |
||||
"time" |
||||
) |
||||
|
||||
// Retry backoff with jitter sleep to prevent overloaded conditions during intervals
|
||||
// https://www.awsarchitectureblog.com/2015/03/backoff.html
|
||||
func RetryBackoff(retry int, minBackoff, maxBackoff time.Duration) time.Duration { |
||||
if retry < 0 { |
||||
retry = 0 |
||||
} |
||||
|
||||
backoff := minBackoff << uint(retry) |
||||
if backoff > maxBackoff || backoff < minBackoff { |
||||
backoff = maxBackoff |
||||
} |
||||
|
||||
if backoff == 0 { |
||||
return 0 |
||||
} |
||||
return time.Duration(rand.Int63n(int64(backoff))) |
||||
} |
||||
@ -0,0 +1,15 @@ |
||||
package internal |
||||
|
||||
import ( |
||||
"fmt" |
||||
"log" |
||||
) |
||||
|
||||
var Logger *log.Logger |
||||
|
||||
func Logf(s string, args ...interface{}) { |
||||
if Logger == nil { |
||||
return |
||||
} |
||||
Logger.Output(2, fmt.Sprintf(s, args...)) |
||||
} |
||||
@ -0,0 +1,60 @@ |
||||
/* |
||||
Copyright 2014 The Camlistore Authors |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
|
||||
package internal |
||||
|
||||
import ( |
||||
"sync" |
||||
"sync/atomic" |
||||
) |
||||
|
||||
// A Once will perform a successful action exactly once.
|
||||
//
|
||||
// Unlike a sync.Once, this Once's func returns an error
|
||||
// and is re-armed on failure.
|
||||
type Once struct { |
||||
m sync.Mutex |
||||
done uint32 |
||||
} |
||||
|
||||
// Do calls the function f if and only if Do has not been invoked
|
||||
// without error for this instance of Once. In other words, given
|
||||
// var once Once
|
||||
// if once.Do(f) is called multiple times, only the first call will
|
||||
// invoke f, even if f has a different value in each invocation unless
|
||||
// f returns an error. A new instance of Once is required for each
|
||||
// function to execute.
|
||||
//
|
||||
// Do is intended for initialization that must be run exactly once. Since f
|
||||
// is niladic, it may be necessary to use a function literal to capture the
|
||||
// arguments to a function to be invoked by Do:
|
||||
// err := config.once.Do(func() error { return config.init(filename) })
|
||||
func (o *Once) Do(f func() error) error { |
||||
if atomic.LoadUint32(&o.done) == 1 { |
||||
return nil |
||||
} |
||||
// Slow-path.
|
||||
o.m.Lock() |
||||
defer o.m.Unlock() |
||||
var err error |
||||
if o.done == 0 { |
||||
err = f() |
||||
if err == nil { |
||||
atomic.StoreUint32(&o.done, 1) |
||||
} |
||||
} |
||||
return err |
||||
} |
||||
@ -0,0 +1,93 @@ |
||||
package pool |
||||
|
||||
import ( |
||||
"net" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/go-redis/redis/internal/proto" |
||||
) |
||||
|
||||
var noDeadline = time.Time{} |
||||
|
||||
type Conn struct { |
||||
netConn net.Conn |
||||
|
||||
rd *proto.Reader |
||||
rdLocked bool |
||||
wr *proto.Writer |
||||
|
||||
InitedAt time.Time |
||||
pooled bool |
||||
usedAt atomic.Value |
||||
} |
||||
|
||||
func NewConn(netConn net.Conn) *Conn { |
||||
cn := &Conn{ |
||||
netConn: netConn, |
||||
} |
||||
cn.rd = proto.NewReader(netConn) |
||||
cn.wr = proto.NewWriter(netConn) |
||||
cn.SetUsedAt(time.Now()) |
||||
return cn |
||||
} |
||||
|
||||
func (cn *Conn) UsedAt() time.Time { |
||||
return cn.usedAt.Load().(time.Time) |
||||
} |
||||
|
||||
func (cn *Conn) SetUsedAt(tm time.Time) { |
||||
cn.usedAt.Store(tm) |
||||
} |
||||
|
||||
func (cn *Conn) SetNetConn(netConn net.Conn) { |
||||
cn.netConn = netConn |
||||
cn.rd.Reset(netConn) |
||||
cn.wr.Reset(netConn) |
||||
} |
||||
|
||||
func (cn *Conn) setReadTimeout(timeout time.Duration) error { |
||||
now := time.Now() |
||||
cn.SetUsedAt(now) |
||||
if timeout > 0 { |
||||
return cn.netConn.SetReadDeadline(now.Add(timeout)) |
||||
} |
||||
return cn.netConn.SetReadDeadline(noDeadline) |
||||
} |
||||
|
||||
func (cn *Conn) setWriteTimeout(timeout time.Duration) error { |
||||
now := time.Now() |
||||
cn.SetUsedAt(now) |
||||
if timeout > 0 { |
||||
return cn.netConn.SetWriteDeadline(now.Add(timeout)) |
||||
} |
||||
return cn.netConn.SetWriteDeadline(noDeadline) |
||||
} |
||||
|
||||
func (cn *Conn) Write(b []byte) (int, error) { |
||||
return cn.netConn.Write(b) |
||||
} |
||||
|
||||
func (cn *Conn) RemoteAddr() net.Addr { |
||||
return cn.netConn.RemoteAddr() |
||||
} |
||||
|
||||
func (cn *Conn) WithReader(timeout time.Duration, fn func(rd *proto.Reader) error) error { |
||||
_ = cn.setReadTimeout(timeout) |
||||
return fn(cn.rd) |
||||
} |
||||
|
||||
func (cn *Conn) WithWriter(timeout time.Duration, fn func(wr *proto.Writer) error) error { |
||||
_ = cn.setWriteTimeout(timeout) |
||||
|
||||
firstErr := fn(cn.wr) |
||||
err := cn.wr.Flush() |
||||
if err != nil && firstErr == nil { |
||||
firstErr = err |
||||
} |
||||
return firstErr |
||||
} |
||||
|
||||
func (cn *Conn) Close() error { |
||||
return cn.netConn.Close() |
||||
} |
||||
@ -0,0 +1,476 @@ |
||||
package pool |
||||
|
||||
import ( |
||||
"errors" |
||||
"net" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/go-redis/redis/internal" |
||||
) |
||||
|
||||
var ErrClosed = errors.New("redis: client is closed") |
||||
var ErrPoolTimeout = errors.New("redis: connection pool timeout") |
||||
|
||||
var timers = sync.Pool{ |
||||
New: func() interface{} { |
||||
t := time.NewTimer(time.Hour) |
||||
t.Stop() |
||||
return t |
||||
}, |
||||
} |
||||
|
||||
// Stats contains pool state information and accumulated stats.
|
||||
type Stats struct { |
||||
Hits uint32 // number of times free connection was found in the pool
|
||||
Misses uint32 // number of times free connection was NOT found in the pool
|
||||
Timeouts uint32 // number of times a wait timeout occurred
|
||||
|
||||
TotalConns uint32 // number of total connections in the pool
|
||||
IdleConns uint32 // number of idle connections in the pool
|
||||
StaleConns uint32 // number of stale connections removed from the pool
|
||||
} |
||||
|
||||
type Pooler interface { |
||||
NewConn() (*Conn, error) |
||||
CloseConn(*Conn) error |
||||
|
||||
Get() (*Conn, error) |
||||
Put(*Conn) |
||||
Remove(*Conn) |
||||
|
||||
Len() int |
||||
IdleLen() int |
||||
Stats() *Stats |
||||
|
||||
Close() error |
||||
} |
||||
|
||||
type Options struct { |
||||
Dialer func() (net.Conn, error) |
||||
OnClose func(*Conn) error |
||||
|
||||
PoolSize int |
||||
MinIdleConns int |
||||
MaxConnAge time.Duration |
||||
PoolTimeout time.Duration |
||||
IdleTimeout time.Duration |
||||
IdleCheckFrequency time.Duration |
||||
} |
||||
|
||||
type ConnPool struct { |
||||
opt *Options |
||||
|
||||
dialErrorsNum uint32 // atomic
|
||||
|
||||
lastDialErrorMu sync.RWMutex |
||||
lastDialError error |
||||
|
||||
queue chan struct{} |
||||
|
||||
connsMu sync.Mutex |
||||
conns []*Conn |
||||
idleConns []*Conn |
||||
poolSize int |
||||
idleConnsLen int |
||||
|
||||
stats Stats |
||||
|
||||
_closed uint32 // atomic
|
||||
} |
||||
|
||||
var _ Pooler = (*ConnPool)(nil) |
||||
|
||||
func NewConnPool(opt *Options) *ConnPool { |
||||
p := &ConnPool{ |
||||
opt: opt, |
||||
|
||||
queue: make(chan struct{}, opt.PoolSize), |
||||
conns: make([]*Conn, 0, opt.PoolSize), |
||||
idleConns: make([]*Conn, 0, opt.PoolSize), |
||||
} |
||||
|
||||
for i := 0; i < opt.MinIdleConns; i++ { |
||||
p.checkMinIdleConns() |
||||
} |
||||
|
||||
if opt.IdleTimeout > 0 && opt.IdleCheckFrequency > 0 { |
||||
go p.reaper(opt.IdleCheckFrequency) |
||||
} |
||||
|
||||
return p |
||||
} |
||||
|
||||
func (p *ConnPool) checkMinIdleConns() { |
||||
if p.opt.MinIdleConns == 0 { |
||||
return |
||||
} |
||||
if p.poolSize < p.opt.PoolSize && p.idleConnsLen < p.opt.MinIdleConns { |
||||
p.poolSize++ |
||||
p.idleConnsLen++ |
||||
go p.addIdleConn() |
||||
} |
||||
} |
||||
|
||||
func (p *ConnPool) addIdleConn() { |
||||
cn, err := p.newConn(true) |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
p.connsMu.Lock() |
||||
p.conns = append(p.conns, cn) |
||||
p.idleConns = append(p.idleConns, cn) |
||||
p.connsMu.Unlock() |
||||
} |
||||
|
||||
func (p *ConnPool) NewConn() (*Conn, error) { |
||||
return p._NewConn(false) |
||||
} |
||||
|
||||
func (p *ConnPool) _NewConn(pooled bool) (*Conn, error) { |
||||
cn, err := p.newConn(pooled) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
p.connsMu.Lock() |
||||
p.conns = append(p.conns, cn) |
||||
if pooled { |
||||
if p.poolSize < p.opt.PoolSize { |
||||
p.poolSize++ |
||||
} else { |
||||
cn.pooled = false |
||||
} |
||||
} |
||||
p.connsMu.Unlock() |
||||
return cn, nil |
||||
} |
||||
|
||||
func (p *ConnPool) newConn(pooled bool) (*Conn, error) { |
||||
if p.closed() { |
||||
return nil, ErrClosed |
||||
} |
||||
|
||||
if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.opt.PoolSize) { |
||||
return nil, p.getLastDialError() |
||||
} |
||||
|
||||
netConn, err := p.opt.Dialer() |
||||
if err != nil { |
||||
p.setLastDialError(err) |
||||
if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.opt.PoolSize) { |
||||
go p.tryDial() |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
cn := NewConn(netConn) |
||||
cn.pooled = pooled |
||||
return cn, nil |
||||
} |
||||
|
||||
func (p *ConnPool) tryDial() { |
||||
for { |
||||
if p.closed() { |
||||
return |
||||
} |
||||
|
||||
conn, err := p.opt.Dialer() |
||||
if err != nil { |
||||
p.setLastDialError(err) |
||||
time.Sleep(time.Second) |
||||
continue |
||||
} |
||||
|
||||
atomic.StoreUint32(&p.dialErrorsNum, 0) |
||||
_ = conn.Close() |
||||
return |
||||
} |
||||
} |
||||
|
||||
func (p *ConnPool) setLastDialError(err error) { |
||||
p.lastDialErrorMu.Lock() |
||||
p.lastDialError = err |
||||
p.lastDialErrorMu.Unlock() |
||||
} |
||||
|
||||
func (p *ConnPool) getLastDialError() error { |
||||
p.lastDialErrorMu.RLock() |
||||
err := p.lastDialError |
||||
p.lastDialErrorMu.RUnlock() |
||||
return err |
||||
} |
||||
|
||||
// Get returns existed connection from the pool or creates a new one.
|
||||
func (p *ConnPool) Get() (*Conn, error) { |
||||
if p.closed() { |
||||
return nil, ErrClosed |
||||
} |
||||
|
||||
err := p.waitTurn() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
for { |
||||
p.connsMu.Lock() |
||||
cn := p.popIdle() |
||||
p.connsMu.Unlock() |
||||
|
||||
if cn == nil { |
||||
break |
||||
} |
||||
|
||||
if p.isStaleConn(cn) { |
||||
_ = p.CloseConn(cn) |
||||
continue |
||||
} |
||||
|
||||
atomic.AddUint32(&p.stats.Hits, 1) |
||||
return cn, nil |
||||
} |
||||
|
||||
atomic.AddUint32(&p.stats.Misses, 1) |
||||
|
||||
newcn, err := p._NewConn(true) |
||||
if err != nil { |
||||
p.freeTurn() |
||||
return nil, err |
||||
} |
||||
|
||||
return newcn, nil |
||||
} |
||||
|
||||
func (p *ConnPool) getTurn() { |
||||
p.queue <- struct{}{} |
||||
} |
||||
|
||||
func (p *ConnPool) waitTurn() error { |
||||
select { |
||||
case p.queue <- struct{}{}: |
||||
return nil |
||||
default: |
||||
timer := timers.Get().(*time.Timer) |
||||
timer.Reset(p.opt.PoolTimeout) |
||||
|
||||
select { |
||||
case p.queue <- struct{}{}: |
||||
if !timer.Stop() { |
||||
<-timer.C |
||||
} |
||||
timers.Put(timer) |
||||
return nil |
||||
case <-timer.C: |
||||
timers.Put(timer) |
||||
atomic.AddUint32(&p.stats.Timeouts, 1) |
||||
return ErrPoolTimeout |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (p *ConnPool) freeTurn() { |
||||
<-p.queue |
||||
} |
||||
|
||||
func (p *ConnPool) popIdle() *Conn { |
||||
if len(p.idleConns) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
idx := len(p.idleConns) - 1 |
||||
cn := p.idleConns[idx] |
||||
p.idleConns = p.idleConns[:idx] |
||||
p.idleConnsLen-- |
||||
p.checkMinIdleConns() |
||||
return cn |
||||