chore: update github.com/couchbase/gomemcached and github.com/couchbase/go-couchbase (#9419)

pull/9421/head
Antoine GIRARD 3 years ago committed by techknowlogick
parent 8873a80276
commit 559fb6ccf0
  1. 2
      go.mod
  2. 4
      go.sum
  3. 123
      vendor/github.com/couchbase/gomemcached/client/collections_filter.go
  4. 30
      vendor/github.com/couchbase/gomemcached/client/mc.go
  5. 346
      vendor/github.com/couchbase/gomemcached/client/upr_event.go
  6. 248
      vendor/github.com/couchbase/gomemcached/client/upr_feed.go
  7. 381
      vendor/github.com/couchbase/gomemcached/flexibleFraming.go
  8. 8
      vendor/github.com/couchbase/gomemcached/mc_constants.go
  9. 359
      vendor/github.com/couchbase/gomemcached/mc_req.go
  10. 15
      vendor/github.com/couchbase/gomemcached/mc_res.go
  11. 249
      vendor/github.com/couchbase/goutils/LICENSE.md
  12. 4
      vendor/modules.txt

@ -24,6 +24,8 @@ require (
github.com/blevesearch/go-porterstemmer v1.0.2 // indirect
github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f // indirect
github.com/boombuler/barcode v0.0.0-20161226211916-fe0f26ff6d26 // indirect
github.com/couchbase/gomemcached v0.0.0-20191004160342-7b5da2ec40b2 // indirect
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 // indirect
github.com/couchbase/vellum v0.0.0-20190829182332-ef2e028c01fd // indirect
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect

@ -97,8 +97,12 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d h1:XMf4E1U+b9E3ElF0mjvfXZdflBRZz4gLp16nQ/QSHQM=
github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
github.com/couchbase/gomemcached v0.0.0-20191004160342-7b5da2ec40b2 h1:vZryARwW4PSFXd9arwegEywvMTvPuXL3/oa+4L5NTe8=
github.com/couchbase/gomemcached v0.0.0-20191004160342-7b5da2ec40b2/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b h1:bZ9rKU2/V8sY+NulSfxDOnXTWcs1rySqdF1sVepihvo=
github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 h1:0WMIDtuXCKEm4wtAJgAAXa/qtM5O9MariLwgHaRlYmk=
github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
github.com/couchbase/vellum v0.0.0-20190829182332-ef2e028c01fd h1:zeuJhcG3f8eePshH3KxkNE+Xtl53pVln9MOUPMyr/1w=
github.com/couchbase/vellum v0.0.0-20190829182332-ef2e028c01fd/go.mod h1:xbc8Ff/oG7h2ejd7AlwOpfd+6QZntc92ygpAOfGwcKY=
github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7 h1:1XjEY/gnjQ+AfXef2U6dxCquhiRzkEpxZuWqs+QxTL8=

@ -0,0 +1,123 @@
package memcached
import (
"encoding/json"
"fmt"
)
// Collection based filter
type CollectionsFilter struct {
ManifestUid uint64
UseManifestUid bool
StreamId uint16
UseStreamId bool
// Use either ScopeId OR CollectionsList, not both
CollectionsList []uint32
ScopeId uint32
}
type nonStreamIdNonResumeScopeMeta struct {
ScopeId string `json:"scope"`
}
type nonStreamIdResumeScopeMeta struct {
ManifestId string `json:"uid"`
}
type nonStreamIdNonResumeCollectionsMeta struct {
CollectionsList []string `json:"collections"`
}
type nonStreamIdResumeCollectionsMeta struct {
ManifestId string `json:"uid"`
CollectionsList []string `json:"collections"`
}
type streamIdNonResumeCollectionsMeta struct {
CollectionsList []string `json:"collections"`
StreamId uint16 `json:"sid"`
}
type streamIdNonResumeScopeMeta struct {
ScopeId string `json:"scope"`
StreamId uint16 `json:"sid"`
}
func (c *CollectionsFilter) IsValid() error {
if c.UseManifestUid {
return fmt.Errorf("Not implemented yet")
}
if len(c.CollectionsList) > 0 && c.ScopeId > 0 {
return fmt.Errorf("Collection list is specified but scope ID is also specified")
}
return nil
}
func (c *CollectionsFilter) outputCollectionsFilterColList() (outputList []string) {
for _, collectionUint := range c.CollectionsList {
outputList = append(outputList, fmt.Sprintf("%x", collectionUint))
}
return
}
func (c *CollectionsFilter) outputScopeId() string {
return fmt.Sprintf("%x", c.ScopeId)
}
func (c *CollectionsFilter) ToStreamReqBody() ([]byte, error) {
if err := c.IsValid(); err != nil {
return nil, err
}
var output interface{}
switch c.UseStreamId {
case true:
switch c.UseManifestUid {
case true:
// TODO
return nil, fmt.Errorf("NotImplemented0")
case false:
switch len(c.CollectionsList) > 0 {
case true:
filter := &streamIdNonResumeCollectionsMeta{
StreamId: c.StreamId,
CollectionsList: c.outputCollectionsFilterColList(),
}
output = *filter
case false:
filter := &streamIdNonResumeScopeMeta{
StreamId: c.StreamId,
ScopeId: c.outputScopeId(),
}
output = *filter
}
}
case false:
switch c.UseManifestUid {
case true:
// TODO
return nil, fmt.Errorf("NotImplemented1")
case false:
switch len(c.CollectionsList) > 0 {
case true:
filter := &nonStreamIdNonResumeCollectionsMeta{
CollectionsList: c.outputCollectionsFilterColList(),
}
output = *filter
case false:
output = nonStreamIdNonResumeScopeMeta{ScopeId: c.outputScopeId()}
}
}
}
data, err := json.Marshal(output)
if err != nil {
return nil, err
} else {
return data, nil
}
}

@ -28,10 +28,12 @@ type ClientIface interface {
CASNext(vb uint16, k string, exp int, state *CASState) bool
CAS(vb uint16, k string, f CasFunc, initexp int) (*gomemcached.MCResponse, error)
CollectionsGetCID(scope string, collection string) (*gomemcached.MCResponse, error)
CollectionEnabled() bool
Close() error
Decr(vb uint16, key string, amt, def uint64, exp int) (uint64, error)
Del(vb uint16, key string) (*gomemcached.MCResponse, error)
EnableMutationToken() (*gomemcached.MCResponse, error)
EnableFeatures(features Features) (*gomemcached.MCResponse, error)
Get(vb uint16, key string) (*gomemcached.MCResponse, error)
GetCollectionsManifest() (*gomemcached.MCResponse, error)
GetFromCollection(vb uint16, cid uint32, key string) (*gomemcached.MCResponse, error)
@ -76,9 +78,12 @@ var Healthy uint32 = 1
type Features []Feature
type Feature uint16
const FeatureMutationToken = Feature(0x04)
const FeatureTcpNoDelay = Feature(0x03)
const FeatureMutationToken = Feature(0x04) // XATTR bit in data type field with dcp mutations
const FeatureXattr = Feature(0x06)
const FeatureXerror = Feature(0x07)
const FeatureCollections = Feature(0x12)
const FeatureSnappyCompression = Feature(0x0a)
const FeatureDataType = Feature(0x0b)
type memcachedConnection interface {
@ -96,6 +101,9 @@ type Client struct {
opaque uint32
hdrBuf []byte
featureMtx sync.RWMutex
sentHeloFeatures Features
}
var (
@ -285,6 +293,10 @@ func (c *Client) EnableFeatures(features Features) (*gomemcached.MCResponse, err
binary.BigEndian.PutUint16(payload[len(payload)-2:], uint16(feature))
}
c.featureMtx.Lock()
c.sentHeloFeatures = features
c.featureMtx.Unlock()
return c.Send(&gomemcached.MCRequest{
Opcode: gomemcached.HELLO,
Key: []byte("GoMemcached"),
@ -363,6 +375,18 @@ func (c *Client) CollectionsGetCID(scope string, collection string) (*gomemcache
return res, nil
}
func (c *Client) CollectionEnabled() bool {
c.featureMtx.RLock()
defer c.featureMtx.RUnlock()
for _, feature := range c.sentHeloFeatures {
if feature == FeatureCollections {
return true
}
}
return false
}
// Get the value for a key, and update expiry
func (c *Client) GetAndTouch(vb uint16, key string, exp int) (*gomemcached.MCResponse, error) {
extraBuf := make([]byte, 4)
@ -1138,3 +1162,7 @@ func IfResStatusError(response *gomemcached.MCResponse) bool {
response.Status != gomemcached.SUBDOC_PATH_NOT_FOUND &&
response.Status != gomemcached.SUBDOC_MULTI_PATH_FAILURE_DELETED)
}
func (c *Client) Conn() io.ReadWriteCloser {
return c.conn
}

@ -0,0 +1,346 @@
package memcached
import (
"encoding/binary"
"fmt"
"github.com/couchbase/gomemcached"
"math"
)
type SystemEventType int
const InvalidSysEvent SystemEventType = -1
const (
CollectionCreate SystemEventType = 0
CollectionDrop SystemEventType = iota
CollectionFlush SystemEventType = iota // KV did not implement
ScopeCreate SystemEventType = iota
ScopeDrop SystemEventType = iota
CollectionChanged SystemEventType = iota
)
type ScopeCreateEvent interface {
GetSystemEventName() (string, error)
GetScopeId() (uint32, error)
GetManifestId() (uint64, error)
}
type CollectionCreateEvent interface {
GetSystemEventName() (string, error)
GetScopeId() (uint32, error)
GetCollectionId() (uint32, error)
GetManifestId() (uint64, error)
GetMaxTTL() (uint32, error)
}
type CollectionDropEvent interface {
GetScopeId() (uint32, error)
GetCollectionId() (uint32, error)
GetManifestId() (uint64, error)
}
type ScopeDropEvent interface {
GetScopeId() (uint32, error)
GetManifestId() (uint64, error)
}
type CollectionChangedEvent interface {
GetCollectionId() (uint32, error)
GetManifestId() (uint64, error)
GetMaxTTL() (uint32, error)
}
var ErrorInvalidOp error = fmt.Errorf("Invalid Operation")
var ErrorInvalidVersion error = fmt.Errorf("Invalid version for parsing")
var ErrorValueTooShort error = fmt.Errorf("Value length is too short")
var ErrorNoMaxTTL error = fmt.Errorf("This event has no max TTL")
// UprEvent memcached events for UPR streams.
type UprEvent struct {
Opcode gomemcached.CommandCode // Type of event
Status gomemcached.Status // Response status
VBucket uint16 // VBucket this event applies to
DataType uint8 // data type
Opaque uint16 // 16 MSB of opaque
VBuuid uint64 // This field is set by downstream
Flags uint32 // Item flags
Expiry uint32 // Item expiration time
Key, Value []byte // Item key/value
OldValue []byte // TODO: TBD: old document value
Cas uint64 // CAS value of the item
Seqno uint64 // sequence number of the mutation
RevSeqno uint64 // rev sequence number : deletions
LockTime uint32 // Lock time
MetadataSize uint16 // Metadata size
SnapstartSeq uint64 // start sequence number of this snapshot
SnapendSeq uint64 // End sequence number of the snapshot
SnapshotType uint32 // 0: disk 1: memory
FailoverLog *FailoverLog // Failover log containing vvuid and sequnce number
Error error // Error value in case of a failure
ExtMeta []byte // Extended Metadata
AckSize uint32 // The number of bytes that can be Acked to DCP
SystemEvent SystemEventType // Only valid if IsSystemEvent() is true
SysEventVersion uint8 // Based on the version, the way Extra bytes is parsed is different
ValueLen int // Cache it to avoid len() calls for performance
CollectionId uint64 // Valid if Collection is in use
}
// FailoverLog containing vvuid and sequnce number
type FailoverLog [][2]uint64
func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFromDCP int) *UprEvent {
event := &UprEvent{
Opcode: rq.Opcode,
VBucket: stream.Vbucket,
VBuuid: stream.Vbuuid,
Value: rq.Body,
Cas: rq.Cas,
ExtMeta: rq.ExtMeta,
DataType: rq.DataType,
ValueLen: len(rq.Body),
SystemEvent: InvalidSysEvent,
CollectionId: math.MaxUint64,
}
event.PopulateFieldsBasedOnStreamType(rq, stream.StreamType)
// set AckSize for events that need to be acked to DCP,
// i.e., events with CommandCodes that need to be buffered in DCP
if _, ok := gomemcached.BufferedCommandCodeMap[rq.Opcode]; ok {
event.AckSize = uint32(bytesReceivedFromDCP)
}
// 16 LSBits are used by client library to encode vbucket number.
// 16 MSBits are left for application to multiplex on opaque value.
event.Opaque = appOpaque(rq.Opaque)
if len(rq.Extras) >= uprMutationExtraLen &&
event.Opcode == gomemcached.UPR_MUTATION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.Flags = binary.BigEndian.Uint32(rq.Extras[16:20])
event.Expiry = binary.BigEndian.Uint32(rq.Extras[20:24])
event.LockTime = binary.BigEndian.Uint32(rq.Extras[24:28])
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[28:30])
} else if len(rq.Extras) >= uprDeletetionWithDeletionTimeExtraLen &&
event.Opcode == gomemcached.UPR_DELETION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.Expiry = binary.BigEndian.Uint32(rq.Extras[16:20])
} else if len(rq.Extras) >= uprDeletetionExtraLen &&
event.Opcode == gomemcached.UPR_DELETION ||
event.Opcode == gomemcached.UPR_EXPIRATION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[16:18])
} else if len(rq.Extras) >= uprSnapshotExtraLen &&
event.Opcode == gomemcached.UPR_SNAPSHOT {
event.SnapstartSeq = binary.BigEndian.Uint64(rq.Extras[:8])
event.SnapendSeq = binary.BigEndian.Uint64(rq.Extras[8:16])
event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
} else if event.IsSystemEvent() {
event.PopulateEvent(rq.Extras)
}
return event
}
func (event *UprEvent) PopulateFieldsBasedOnStreamType(rq gomemcached.MCRequest, streamType DcpStreamType) {
switch streamType {
case CollectionsNonStreamId:
switch rq.Opcode {
// Only these will have CID encoded within the key
case gomemcached.UPR_MUTATION,
gomemcached.UPR_DELETION,
gomemcached.UPR_EXPIRATION:
uleb128 := Uleb128(rq.Key)
result, bytesShifted := uleb128.ToUint64(rq.Keylen)
event.CollectionId = result
event.Key = rq.Key[bytesShifted:]
default:
event.Key = rq.Key
}
case CollectionsStreamId:
// TODO - not implemented
fallthrough
case NonCollectionStream:
// Let default behavior be legacy stream type
fallthrough
default:
event.Key = rq.Key
}
}
func (event *UprEvent) String() string {
name := gomemcached.CommandNames[event.Opcode]
if name == "" {
name = fmt.Sprintf("#%d", event.Opcode)
}
return name
}
func (event *UprEvent) IsSnappyDataType() bool {
return event.Opcode == gomemcached.UPR_MUTATION && (event.DataType&SnappyDataType > 0)
}
func (event *UprEvent) IsCollectionType() bool {
return event.IsSystemEvent() || event.CollectionId <= math.MaxUint32
}
func (event *UprEvent) IsSystemEvent() bool {
return event.Opcode == gomemcached.DCP_SYSTEM_EVENT
}
func (event *UprEvent) PopulateEvent(extras []byte) {
if len(extras) < dcpSystemEventExtraLen {
// Wrong length, don't parse
return
}
event.Seqno = binary.BigEndian.Uint64(extras[:8])
event.SystemEvent = SystemEventType(binary.BigEndian.Uint32(extras[8:12]))
var versionTemp uint16 = binary.BigEndian.Uint16(extras[12:14])
event.SysEventVersion = uint8(versionTemp >> 8)
}
func (event *UprEvent) GetSystemEventName() (string, error) {
switch event.SystemEvent {
case CollectionCreate:
fallthrough
case ScopeCreate:
return string(event.Key), nil
default:
return "", ErrorInvalidOp
}
}
func (event *UprEvent) GetManifestId() (uint64, error) {
switch event.SystemEvent {
// Version 0 only checks
case CollectionChanged:
fallthrough
case ScopeDrop:
fallthrough
case ScopeCreate:
fallthrough
case CollectionDrop:
if event.SysEventVersion > 0 {
return 0, ErrorInvalidVersion
}
fallthrough
case CollectionCreate:
// CollectionCreate supports version 1
if event.SysEventVersion > 1 {
return 0, ErrorInvalidVersion
}
if event.ValueLen < 8 {
return 0, ErrorValueTooShort
}
return binary.BigEndian.Uint64(event.Value[0:8]), nil
default:
return 0, ErrorInvalidOp
}
}
func (event *UprEvent) GetCollectionId() (uint32, error) {
switch event.SystemEvent {
case CollectionDrop:
if event.SysEventVersion > 0 {
return 0, ErrorInvalidVersion
}
fallthrough
case CollectionCreate:
if event.SysEventVersion > 1 {
return 0, ErrorInvalidVersion
}
if event.ValueLen < 16 {
return 0, ErrorValueTooShort
}
return binary.BigEndian.Uint32(event.Value[12:16]), nil
case CollectionChanged:
if event.SysEventVersion > 0 {
return 0, ErrorInvalidVersion
}
if event.ValueLen < 12 {
return 0, ErrorValueTooShort
}
return binary.BigEndian.Uint32(event.Value[8:12]), nil
default:
return 0, ErrorInvalidOp
}
}
func (event *UprEvent) GetScopeId() (uint32, error) {
switch event.SystemEvent {
// version 0 checks
case ScopeCreate:
fallthrough
case ScopeDrop:
fallthrough
case CollectionDrop:
if event.SysEventVersion > 0 {
return 0, ErrorInvalidVersion
}
fallthrough
case CollectionCreate:
// CollectionCreate could be either 0 or 1
if event.SysEventVersion > 1 {
return 0, ErrorInvalidVersion
}
if event.ValueLen < 12 {
return 0, ErrorValueTooShort
}
return binary.BigEndian.Uint32(event.Value[8:12]), nil
default:
return 0, ErrorInvalidOp
}
}
func (event *UprEvent) GetMaxTTL() (uint32, error) {
switch event.SystemEvent {
case CollectionCreate:
if event.SysEventVersion < 1 {
return 0, ErrorNoMaxTTL
}
if event.ValueLen < 20 {
return 0, ErrorValueTooShort
}
return binary.BigEndian.Uint32(event.Value[16:20]), nil
case CollectionChanged:
if event.SysEventVersion > 0 {
return 0, ErrorInvalidVersion
}
if event.ValueLen < 16 {
return 0, ErrorValueTooShort
}
return binary.BigEndian.Uint32(event.Value[12:16]), nil
default:
return 0, ErrorInvalidOp
}
}
type Uleb128 []byte
func (u Uleb128) ToUint64(cachedLen int) (result uint64, bytesShifted int) {
var shift uint = 0
for curByte := 0; curByte < cachedLen; curByte++ {
oneByte := u[curByte]
last7Bits := 0x7f & oneByte
result |= uint64(last7Bits) << shift
bytesShifted++
if oneByte&0x80 == 0 {
break
}
shift += 7
}
return
}

@ -19,6 +19,7 @@ const uprMutationExtraLen = 30
const uprDeletetionExtraLen = 18
const uprDeletetionWithDeletionTimeExtraLen = 21
const uprSnapshotExtraLen = 20
const dcpSystemEventExtraLen = 13
const bufferAckThreshold = 0.2
const opaqueOpen = 0xBEAF0001
const opaqueFailover = 0xDEADBEEF
@ -27,32 +28,6 @@ const uprDefaultNoopInterval = 120
// Counter on top of opaqueOpen that others can draw from for open and control msgs
var opaqueOpenCtrlWell uint32 = opaqueOpen
// UprEvent memcached events for UPR streams.
type UprEvent struct {
Opcode gomemcached.CommandCode // Type of event
Status gomemcached.Status // Response status
VBucket uint16 // VBucket this event applies to
DataType uint8 // data type
Opaque uint16 // 16 MSB of opaque
VBuuid uint64 // This field is set by downstream
Flags uint32 // Item flags
Expiry uint32 // Item expiration time
Key, Value []byte // Item key/value
OldValue []byte // TODO: TBD: old document value
Cas uint64 // CAS value of the item
Seqno uint64 // sequence number of the mutation
RevSeqno uint64 // rev sequence number : deletions
LockTime uint32 // Lock time
MetadataSize uint16 // Metadata size
SnapstartSeq uint64 // start sequence number of this snapshot
SnapendSeq uint64 // End sequence number of the snapshot
SnapshotType uint32 // 0: disk 1: memory
FailoverLog *FailoverLog // Failover log containing vvuid and sequnce number
Error error // Error value in case of a failure
ExtMeta []byte
AckSize uint32 // The number of bytes that can be Acked to DCP
}
type PriorityType string
// high > medium > disabled > low
@ -63,13 +38,39 @@ const (
PriorityHigh PriorityType = "high"
)
type DcpStreamType int32
var UninitializedStream DcpStreamType = -1
const (
NonCollectionStream DcpStreamType = 0
CollectionsNonStreamId DcpStreamType = iota
CollectionsStreamId DcpStreamType = iota
)
func (t DcpStreamType) String() string {
switch t {
case UninitializedStream:
return "Un-Initialized Stream"
case NonCollectionStream:
return "Traditional Non-Collection Stream"
case CollectionsNonStreamId:
return "Collections Stream without StreamID"
case CollectionsStreamId:
return "Collection Stream with StreamID"
default:
return "Unknown Stream Type"
}
}
// UprStream is per stream data structure over an UPR Connection.
type UprStream struct {
Vbucket uint16 // Vbucket id
Vbuuid uint64 // vbucket uuid
StartSeq uint64 // start sequence number
EndSeq uint64 // end sequence number
connected bool
Vbucket uint16 // Vbucket id
Vbuuid uint64 // vbucket uuid
StartSeq uint64 // start sequence number
EndSeq uint64 // end sequence number
connected bool
StreamType DcpStreamType
}
type FeedState int
@ -113,6 +114,7 @@ type UprFeatures struct {
IncludeDeletionTime bool
DcpPriority PriorityType
EnableExpiry bool
EnableStreamId bool
}
/**
@ -274,9 +276,15 @@ type UprFeed struct {
// if flag is true, upr feed will use ack from client to determine whether/when to send ack to DCP
// if flag is false, upr feed will track how many bytes it has sent to client
// and use that to determine whether/when to send ack to DCP
ackByClient bool
feedState FeedState
muFeedState sync.RWMutex
ackByClient bool
feedState FeedState
muFeedState sync.RWMutex
activatedFeatures UprFeatures
collectionEnabled bool // This is needed separately because parsing depends on this
// DCP StreamID allows multiple filtered collection streams to share a single DCP Stream
// It is not allowed once a regular/legacy stream was started originally
streamsType DcpStreamType
initStreamTypeOnce sync.Once
}
// Exported interface - to allow for mocking
@ -296,6 +304,9 @@ type UprFeedIface interface {
UprRequestStream(vbno, opaqueMSB uint16, flags uint32, vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error
// Set DCP priority on an existing DCP connection. The command is sent asynchronously without waiting for a response
SetPriorityAsync(p PriorityType) error
// Various Collection-Type RequestStreams
UprRequestCollectionsStream(vbno, opaqueMSB uint16, flags uint32, vbuuid, startSeq, endSeq, snapStart, snapEnd uint64, filter *CollectionsFilter) error
}
type UprStats struct {
@ -305,9 +316,6 @@ type UprStats struct {
TotalSnapShot uint64
}
// FailoverLog containing vvuid and sequnce number
type FailoverLog [][2]uint64
// error codes
var ErrorInvalidLog = errors.New("couchbase.errorInvalidLog")
@ -320,76 +328,6 @@ func (flogp *FailoverLog) Latest() (vbuuid, seqno uint64, err error) {
return vbuuid, seqno, ErrorInvalidLog
}
func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFromDCP int) *UprEvent {
event := &UprEvent{
Opcode: rq.Opcode,
VBucket: stream.Vbucket,
VBuuid: stream.Vbuuid,
Key: rq.Key,
Value: rq.Body,
Cas: rq.Cas,
ExtMeta: rq.ExtMeta,
DataType: rq.DataType,
}
// set AckSize for events that need to be acked to DCP,
// i.e., events with CommandCodes that need to be buffered in DCP
if _, ok := gomemcached.BufferedCommandCodeMap[rq.Opcode]; ok {
event.AckSize = uint32(bytesReceivedFromDCP)
}
// 16 LSBits are used by client library to encode vbucket number.
// 16 MSBits are left for application to multiplex on opaque value.
event.Opaque = appOpaque(rq.Opaque)
if len(rq.Extras) >= uprMutationExtraLen &&
event.Opcode == gomemcached.UPR_MUTATION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.Flags = binary.BigEndian.Uint32(rq.Extras[16:20])
event.Expiry = binary.BigEndian.Uint32(rq.Extras[20:24])
event.LockTime = binary.BigEndian.Uint32(rq.Extras[24:28])
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[28:30])
} else if len(rq.Extras) >= uprDeletetionWithDeletionTimeExtraLen &&
event.Opcode == gomemcached.UPR_DELETION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.Expiry = binary.BigEndian.Uint32(rq.Extras[16:20])
} else if len(rq.Extras) >= uprDeletetionExtraLen &&
event.Opcode == gomemcached.UPR_DELETION ||
event.Opcode == gomemcached.UPR_EXPIRATION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[16:18])
} else if len(rq.Extras) >= uprSnapshotExtraLen &&
event.Opcode == gomemcached.UPR_SNAPSHOT {
event.SnapstartSeq = binary.BigEndian.Uint64(rq.Extras[:8])
event.SnapendSeq = binary.BigEndian.Uint64(rq.Extras[8:16])
event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
}
return event
}
func (event *UprEvent) String() string {
name := gomemcached.CommandNames[event.Opcode]
if name == "" {
name = fmt.Sprintf("#%d", event.Opcode)
}
return name
}
func (event *UprEvent) IsSnappyDataType() bool {
return event.Opcode == gomemcached.UPR_MUTATION && (event.DataType&SnappyDataType > 0)
}
func (feed *UprFeed) sendCommands(mc *Client) {
transmitCh := feed.transmitCh
transmitCl := feed.transmitCl
@ -420,6 +358,10 @@ func (feed *UprFeed) activateStream(vbno, opaque uint16, stream *UprStream) erro
feed.muVbstreams.Lock()
defer feed.muVbstreams.Unlock()
if feed.collectionEnabled {
stream.StreamType = feed.streamsType
}
// Set this stream as the officially connected stream for this vb
stream.connected = true
feed.vbstreams[vbno] = stream
@ -440,14 +382,15 @@ func (mc *Client) NewUprFeed() (*UprFeed, error) {
}
func (mc *Client) NewUprFeedWithConfig(ackByClient bool) (*UprFeed, error) {
feed := &UprFeed{
conn: mc,
closer: make(chan bool, 1),
vbstreams: make(map[uint16]*UprStream),
transmitCh: make(chan *gomemcached.MCRequest),
transmitCl: make(chan bool),
ackByClient: ackByClient,
conn: mc,
closer: make(chan bool, 1),
vbstreams: make(map[uint16]*UprStream),
transmitCh: make(chan *gomemcached.MCRequest),
transmitCl: make(chan bool),
ackByClient: ackByClient,
collectionEnabled: mc.CollectionEnabled(),
streamsType: UninitializedStream,
}
feed.negotiator.initialize()
@ -642,7 +585,22 @@ func (feed *UprFeed) uprOpen(name string, sequence uint32, bufSize uint32, featu
activatedFeatures.EnableExpiry = true
}
if features.EnableStreamId {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_CONTROL,
Key: []byte("enable_stream_id"),
Body: []byte("true"),
Opaque: getUprOpenCtrlOpaque(),
}
err = sendMcRequestSync(feed.conn, rq)
if err != nil {
return
}
activatedFeatures.EnableStreamId = true
}
// everything is ok so far, set upr feed to open state
feed.activatedFeatures = activatedFeatures
feed.setOpen()
return
}
@ -689,10 +647,60 @@ func (mc *Client) UprGetFailoverLog(
func (feed *UprFeed) UprRequestStream(vbno, opaqueMSB uint16, flags uint32,
vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error {
return feed.UprRequestCollectionsStream(vbno, opaqueMSB, flags, vuuid, startSequence, endSequence, snapStart, snapEnd, nil)
}
func (feed *UprFeed) initStreamType(filter *CollectionsFilter) (err error) {
if filter != nil && filter.UseStreamId && !feed.activatedFeatures.EnableStreamId {
err = fmt.Errorf("Cannot use streamID based filter if the feed was not started with the streamID feature")
return
}
streamInitFunc := func() {
if feed.streamsType != UninitializedStream {
// Shouldn't happen
err = fmt.Errorf("The current feed has already been started in %v mode", feed.streamsType.String())
} else {
if !feed.collectionEnabled {
feed.streamsType = NonCollectionStream
} else {
if filter != nil && filter.UseStreamId {
feed.streamsType = CollectionsStreamId
} else {
feed.streamsType = CollectionsNonStreamId
}
}
}
}
feed.initStreamTypeOnce.Do(streamInitFunc)
return
}
func (feed *UprFeed) UprRequestCollectionsStream(vbno, opaqueMSB uint16, flags uint32,
vbuuid, startSequence, endSequence, snapStart, snapEnd uint64, filter *CollectionsFilter) error {
err := feed.initStreamType(filter)
if err != nil {
return err
}
var mcRequestBody []byte
if filter != nil {
err = filter.IsValid()
if err != nil {
return err
}
mcRequestBody, err = filter.ToStreamReqBody()
if err != nil {
return err
}
}
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_STREAMREQ,
VBucket: vbno,
Opaque: composeOpaque(vbno, opaqueMSB),
Body: mcRequestBody,
}
rq.Extras = make([]byte, 48) // #Extras
@ -700,15 +708,15 @@ func (feed *UprFeed) UprRequestStream(vbno, opaqueMSB uint16, flags uint32,
binary.BigEndian.PutUint32(rq.Extras[4:8], uint32(0))
binary.BigEndian.PutUint64(rq.Extras[8:16], startSequence)
binary.BigEndian.PutUint64(rq.Extras[16:24], endSequence)
binary.BigEndian.PutUint64(rq.Extras[24:32], vuuid)
binary.BigEndian.PutUint64(rq.Extras[24:32], vbuuid)
binary.BigEndian.PutUint64(rq.Extras[32:40], snapStart)
binary.BigEndian.PutUint64(rq.Extras[40:48], snapEnd)
feed.negotiator.registerRequest(vbno, opaqueMSB, vuuid, startSequence, endSequence)
feed.negotiator.registerRequest(vbno, opaqueMSB, vbuuid, startSequence, endSequence)
// Any client that has ever called this method, regardless of return code,
// should expect a potential UPR_CLOSESTREAM message due to this new map entry prior to Transmit.
if err := feed.conn.Transmit(rq); err != nil {
if err = feed.conn.Transmit(rq); err != nil {
logging.Errorf("Error in StreamRequest %s", err.Error())
// If an error occurs during transmit, then the UPRFeed will keep the stream
// in the vbstreams map. This is to prevent nil lookup from any previously
@ -973,6 +981,12 @@ loop:
if err := feed.conn.TransmitResponse(noop); err != nil {
logging.Warnf("failed to transmit command %s. Error %s", noop.Opcode.String(), err.Error())
}
case gomemcached.DCP_SYSTEM_EVENT:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
event = makeUprEvent(pkt, stream, bytes)
default:
logging.Infof("Recived an unknown response for vbucket %d", vb)
}

@ -0,0 +1,381 @@
package gomemcached
import (
"encoding/binary"
"fmt"
)
type FrameObjType int
const (
FrameBarrier FrameObjType = iota
FrameDurability FrameObjType = iota
FrameDcpStreamId FrameObjType = iota
FrameOpenTracing FrameObjType = iota
)
type FrameInfo struct {
ObjId FrameObjType
ObjLen int
ObjData []byte
}
var ErrorInvalidOp error = fmt.Errorf("Specified method is not applicable")
var ErrorObjLenNotMatch error = fmt.Errorf("Object length does not match data")
func (f *FrameInfo) Validate() error {
switch f.ObjId {
case FrameBarrier:
if f.ObjLen != 0 {
return fmt.Errorf("Invalid FrameBarrier - length is %v\n", f.ObjLen)
} else if f.ObjLen != len(f.ObjData) {
return ErrorObjLenNotMatch
}
case FrameDurability:
if f.ObjLen != 1 && f.ObjLen != 3 {
return fmt.Errorf("Invalid FrameDurability - length is %v\n", f.ObjLen)
} else if f.ObjLen != len(f.ObjData) {
return ErrorObjLenNotMatch
}
case FrameDcpStreamId:
if f.ObjLen != 2 {
return fmt.Errorf("Invalid FrameDcpStreamId - length is %v\n", f.ObjLen)
} else if f.ObjLen != len(f.ObjData) {
return ErrorObjLenNotMatch
}
case FrameOpenTracing:
if f.ObjLen == 0 {
return fmt.Errorf("Invalid FrameOpenTracing - length must be > 0")
} else if f.ObjLen != len(f.ObjData) {
return ErrorObjLenNotMatch
}
default:
return fmt.Errorf("Unknown FrameInfo type")
}
return nil
}
func (f *FrameInfo) GetStreamId() (uint16, error) {
if f.ObjId != FrameDcpStreamId {
return 0, ErrorInvalidOp
}
var output uint16
output = uint16(f.ObjData[0])
output = output << 8
output |= uint16(f.ObjData[1])
return output, nil
}
type DurabilityLvl uint8
const (
DuraInvalid DurabilityLvl = iota // Not used (0x0)
DuraMajority DurabilityLvl = iota // (0x01)
DuraMajorityAndPersistOnMaster DurabilityLvl = iota // (0x02)
DuraPersistToMajority DurabilityLvl = iota // (0x03)
)
func (f *FrameInfo) GetDurabilityRequirements() (lvl DurabilityLvl, timeoutProvided bool, timeoutMs uint16, err error) {
if f.ObjId != FrameDurability {
err = ErrorInvalidOp
return
}
if f.ObjLen != 1 && f.ObjLen != 3 {
err = ErrorObjLenNotMatch
return
}
lvl = DurabilityLvl(uint8(f.ObjData[0]))
if f.ObjLen == 3 {
timeoutProvided = true
timeoutMs = binary.BigEndian.Uint16(f.ObjData[1:2])
}
return
}
func incrementMarker(bitsToBeIncremented, byteIncrementCnt *int, framingElen, curObjIdx int) (int, error) {
for *bitsToBeIncremented >= 8 {
*byteIncrementCnt++
*bitsToBeIncremented -= 8
}
marker := curObjIdx + *byteIncrementCnt
if marker > framingElen {
return -1, fmt.Errorf("Out of bounds")
}
return marker, nil
}
// Right now, halfByteRemaining will always be false, because ObjID and Len haven't gotten that large yet
func (f *FrameInfo) Bytes() (output []byte, halfByteRemaining bool) {
// ObjIdentifier - 4 bits + ObjLength - 4 bits
var idAndLen uint8
idAndLen |= uint8(f.ObjId) << 4
idAndLen |= uint8(f.ObjLen)
output = append(output, byte(idAndLen))
// Rest is Data
output = append(output, f.ObjData...)
return
}
func parseFrameInfoObjects(buf []byte, framingElen int) (objs []FrameInfo, err error, halfByteRemaining bool) {
var curObjIdx int
var byteIncrementCnt int
var bitsToBeIncremented int
var marker int
// Parse frameInfo objects
for curObjIdx = 0; curObjIdx < framingElen; curObjIdx += byteIncrementCnt {
byteIncrementCnt = 0
var oneFrameObj FrameInfo
// First get the objId
// -------------------------
var objId int
var objHeader uint8 = buf[curObjIdx]
var objIdentifierRaw uint8
if bitsToBeIncremented == 0 {
// ObjHeader
// 0 1 2 3 4 5 6 7
// ^-----^
// ObjIdentifierRaw
objIdentifierRaw = (objHeader & 0xf0) >> 4
} else {
// ObjHeader
// 0 1 2 3 4 5 6 7
// ^-----^
// ObjIdentifierRaw
objIdentifierRaw = (objHeader & 0x0f)
}
bitsToBeIncremented += 4
marker, err = incrementMarker(&bitsToBeIncremented, &byteIncrementCnt, framingElen, curObjIdx)
if err != nil {
return
}
// Value is 0-14
objId = int(objIdentifierRaw & 0xe)
// If bit 15 is set, ID is 15 + value of next byte
if objIdentifierRaw&0x1 > 0 {
if bitsToBeIncremented > 0 {
// ObjHeader
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// ^-----^ ^---------------^
// ObjId1 Extension
// ^ marker
buffer := uint16(buf[marker])
buffer = buffer << 8
buffer |= uint16(buf[marker+1])
var extension uint8 = uint8(buffer & 0xff0 >> 4)
objId += int(extension)
} else {
// ObjHeader
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// ^-----^ ^-------------------^
// ObjId1 extension
// ^ marker
var extension uint8 = uint8(buf[marker])
objId += int(extension)
}
bitsToBeIncremented += 8
}
marker, err = incrementMarker(&bitsToBeIncremented, &byteIncrementCnt, framingElen, curObjIdx)
if err != nil {
return
}
oneFrameObj.ObjId = FrameObjType(objId)
// Then get the obj length
// -------------------------
var objLenRaw uint8
var objLen int
if bitsToBeIncremented > 0 {
// ObjHeader
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// ^ ^---------^
// marker objLen
objLenRaw = uint8(buf[marker]) & 0x0f
} else {
// ObjHeader
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
// ^--------^
// objLen
// ^ marker
objLenRaw = uint8(buf[marker]) & 0xf0 >> 4
}
bitsToBeIncremented += 4
marker, err = incrementMarker(&bitsToBeIncremented, &byteIncrementCnt, framingElen, curObjIdx)
if err != nil {
return
}
// Length is 0-14
objLen = int(objLenRaw & 0xe)
// If bit 15 is set, lenghth is 15 + value of next byte
if objLenRaw&0x1 > 0 {
if bitsToBeIncremented == 0 {
// ObjHeader
// 12 13 14 15 16 17 18 19 20 21 22 23
// ^---------^ ^--------------------^
// objLen extension
// ^ marker
var extension uint8 = uint8(buf[marker])
objLen += int(extension)
} else {
// ObjHeader
// 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
// ^--------^ ^---------------------^
// objLen extension
// ^ marker var buffer uint16
buffer := uint16(buf[marker])
buffer = buffer << 8
buffer |= uint16(buf[marker+1])
var extension uint8 = uint8(buffer & 0xff0 >> 4)
objLen += int(extension)
}
bitsToBeIncremented += 8
}
marker, err = incrementMarker(&bitsToBeIncremented, &byteIncrementCnt, framingElen, curObjIdx)
if err != nil {
return
}
oneFrameObj.ObjLen = objLen
// The rest is N-bytes of data based on the length
if bitsToBeIncremented == 0 {
// No weird alignment needed
oneFrameObj.ObjData = buf[marker : marker+objLen]
} else {
// 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
// ^--------^ ^---------------------^ ^--------->
// objLen extension data
// ^ marker
oneFrameObj.ObjData = ShiftByteSliceLeft4Bits(buf[marker : marker+objLen+1])
}
err = oneFrameObj.Validate()
if err != nil {
return
}
objs = append(objs, oneFrameObj)
bitsToBeIncremented += 8 * objLen
marker, err = incrementMarker(&bitsToBeIncremented, &byteIncrementCnt, framingElen, curObjIdx)
}
if bitsToBeIncremented > 0 {
halfByteRemaining = true
}
return
}
func ShiftByteSliceLeft4Bits(slice []byte) (replacement []byte) {
var buffer uint16
var i int
sliceLen := len(slice)
if sliceLen < 2 {
// Let's not shift less than 16 bits
return
}
replacement = make([]byte, sliceLen, cap(slice))
for i = 0; i < sliceLen-1; i++ {
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// ^-----^ ^---------------^ ^-----------
// garbage data byte 0 data byte 1
buffer = uint16(slice[i])
buffer = buffer << 8
buffer |= uint16(slice[i+1])
replacement[i] = uint8(buffer & 0xff0 >> 4)
}
if i < sliceLen {
lastByte := slice[sliceLen-1]
lastByte = lastByte << 4
replacement[i] = lastByte
}
return
}
// The following is used to theoretically support frameInfo ObjID extensions
// for completeness, but they are not very efficient though
func ShiftByteSliceRight4Bits(slice []byte) (replacement []byte) {
var buffer uint16
var i int
var leftovers uint8 // 4 bits only
var replacementUnit uint16
var first bool = true
var firstLeftovers uint8
var lastLeftovers uint8
sliceLen := len(slice)
if sliceLen < 2 {
// Let's not shift less than 16 bits
return
}
if slice[sliceLen-1]&0xf == 0 {
replacement = make([]byte, sliceLen, cap(slice))
} else {
replacement = make([]byte, sliceLen+1, cap(slice)+1)
}
for i = 0; i < sliceLen-1; i++ {
buffer = binary.BigEndian.Uint16(slice[i : i+2])
// (buffer)
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// ^-------------^ ^-------------------^
// data byte 0 data byte 1
//
// into
//
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
// ^-----^ ^---------------^ ^--------------------^ ^----------^
// zeroes data byte 0 data byte 1 zeroes
if first {
// The leftover OR'ing will overwrite the first 4 bits of data byte 0. Save them
firstLeftovers = uint8(buffer & 0xf000 >> 12)
first = false
}
replacementUnit = 0
replacementUnit |= uint16(leftovers) << 12
replacementUnit |= (buffer & 0xff00) >> 4 // data byte 0
replacementUnit |= buffer & 0xff >> 4 // data byte 1 first 4 bits
lastLeftovers = uint8(buffer&0xf) << 4
replacement[i+1] = byte(replacementUnit)
leftovers = uint8((buffer & 0x000f) << 4)
}
replacement[0] = byte(uint8(replacement[0]) | firstLeftovers)
if lastLeftovers > 0 {
replacement[sliceLen] = byte(lastLeftovers)
}
return
}
func Merge2HalfByteSlices(src1, src2 []byte) (output []byte) {
src1Len := len(src1)
src2Len := len(src2)
output = make([]byte, src1Len+src2Len-1)
var mergeByte uint8 = src1[src1Len-1]
mergeByte |= uint8(src2[0])
copy(output, src1)
copy(output[src1Len:], src2[1:])
output[src1Len-1] = byte(mergeByte)
return
}

@ -6,8 +6,10 @@ import (
)
const (
REQ_MAGIC = 0x80
RES_MAGIC = 0x81
REQ_MAGIC = 0x80
RES_MAGIC = 0x81
FLEX_MAGIC = 0x08
FLEX_RES_MAGIC = 0x18
)
// CommandCode for memcached packets.
@ -99,6 +101,8 @@ const (
SUBDOC_GET = CommandCode(0xc5) // Get subdoc. Returns with xattrs
SUBDOC_MULTI_LOOKUP = CommandCode(0xd0) // Multi lookup. Doc xattrs and meta.
DCP_SYSTEM_EVENT = CommandCode(0x5f) // A system event has occurred
)
// command codes that are counted toward DCP control buffer

@ -25,11 +25,17 @@ type MCRequest struct {
Extras, Key, Body, ExtMeta []byte
// Datatype identifier
DataType uint8
// len() calls are expensive - cache this in case for collection
Keylen int
// Flexible Framing Extras
FramingExtras []FrameInfo
// Stored length of incoming framing extras
FramingElen int
}
// Size gives the number of bytes this request requires.
func (req *MCRequest) Size() int {
return HDR_LEN + len(req.Extras) + len(req.Key) + len(req.Body) + len(req.ExtMeta)
return HDR_LEN + len(req.Extras) + len(req.Key) + len(req.Body) + len(req.ExtMeta) + req.FramingElen
}
// A debugging string representation of this request
@ -38,7 +44,23 @@ func (req MCRequest) String() string {
req.Opcode, len(req.Body), req.Key)
}
func (req *MCRequest) fillHeaderByt