gitea/vendor/github.com/src-d/enry/v2/internal/tokenizer/common.go

8 lines
313 B
Go

// Package tokenizer implements file tokenization used by the enry content
// classifier. This package is an implementation detail of enry and should not
// be imported by other packages.
package tokenizer
// ByteLimit defines the maximum prefix of an input text that will be tokenized.
const ByteLimit = 100000