Restrict repository indexing by file extension

This commit is contained in:
Guillermo Prandi 2019-08-06 01:01:28 -03:00
parent 026696b87a
commit a85393b655
3 changed files with 35 additions and 0 deletions

View File

@ -177,6 +177,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
- `REPO_INDEXER_EXTENSIONS`: **empty**: A comma separated list of file extensions to include/exclude from the index; a \`.' matches files with no extension. An empty list means include all files, disregarding `REPO_EXTENSIONS_LIST_EXCLUDE`.
- `REPO_EXTENSIONS_LIST_EXCLUDE`: **false**: If true, `REPO_INDEXER_EXTENSIONS` are the file extensions to exclude rather than include in the index.
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed. - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.

View File

@ -232,6 +232,17 @@ func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) e
} }
func isIndexable(entry *git.TreeEntry) bool { func isIndexable(entry *git.TreeEntry) bool {
if setting.Indexer.FileExtensions != nil {
var ext string
parts := strings.Split(entry.Name(), ".")
cnt := len(parts)
if cnt > 1 {
ext = strings.ToLower(parts[cnt-1])
}
if setting.Indexer.FileExtensions[ext] == setting.Indexer.ExcludeExtensions {
return false
}
}
return entry.IsRegular() || entry.IsExecutable() return entry.IsRegular() || entry.IsExecutable()
} }

View File

@ -7,6 +7,7 @@ package setting
import ( import (
"path" "path"
"path/filepath" "path/filepath"
"strings"
) )
// enumerates all the indexer queue types // enumerates all the indexer queue types
@ -29,6 +30,8 @@ var (
IssueQueueDir string IssueQueueDir string
IssueQueueConnStr string IssueQueueConnStr string
IssueQueueBatchNumber int IssueQueueBatchNumber int
FileExtensions map[string]bool
ExcludeExtensions bool
}{ }{
IssueType: "bleve", IssueType: "bleve",
IssuePath: "indexers/issues.bleve", IssuePath: "indexers/issues.bleve",
@ -51,6 +54,9 @@ func newIndexerService() {
if !filepath.IsAbs(Indexer.RepoPath) { if !filepath.IsAbs(Indexer.RepoPath) {
Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath) Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
} }
Indexer.FileExtensions = extensionsFromString(sec.Key("REPO_INDEXER_EXTENSIONS").MustString(""))
Indexer.ExcludeExtensions = sec.Key("REPO_EXTENSIONS_LIST_EXCLUDE").MustBool(false)
Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType) Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)
@ -58,3 +64,19 @@ func newIndexerService() {
Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, "")) Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, ""))
Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20) Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)
} }
func extensionsFromString(from string) map[string]bool {
extmap := make(map[string]bool)
for _, ext := range strings.Split(strings.ToLower(from), ",") {
ext = strings.TrimSpace(ext)
if ext == "." {
extmap[""] = true
} else if ext != "" && !strings.Contains(ext, ".") {
extmap[ext] = true
}
}
if len(extmap) == 0 {
return nil
}
return extmap
}