From a85393b6552d5bef79cc9b93a3d3db65ca4d904b Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Tue, 6 Aug 2019 01:01:28 -0300 Subject: [PATCH] Restrict repository indexing by file extension --- .../doc/advanced/config-cheat-sheet.en-us.md | 2 ++ models/repo_indexer.go | 11 ++++++++++ modules/setting/indexer.go | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 6f3bc465a6..affb90eef7 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -177,6 +177,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. +- `REPO_INDEXER_EXTENSIONS`: **empty**: A comma separated list of file extensions to include/exclude from the index; a \`.' matches files with no extension. An empty list means include all files, disregarding `REPO_EXTENSIONS_LIST_EXCLUDE`. +- `REPO_EXTENSIONS_LIST_EXCLUDE`: **false**: If true, `REPO_INDEXER_EXTENSIONS` are the file extensions to exclude rather than include in the index. - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed. diff --git a/models/repo_indexer.go b/models/repo_indexer.go index c991b1aac9..d54b9e397e 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -232,6 +232,17 @@ func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) e } func isIndexable(entry *git.TreeEntry) bool { + if setting.Indexer.FileExtensions != nil { + var ext string + parts := strings.Split(entry.Name(), ".") + cnt := len(parts) + if cnt > 1 { + ext = strings.ToLower(parts[cnt-1]) + } + if setting.Indexer.FileExtensions[ext] == setting.Indexer.ExcludeExtensions { + return false + } + } return entry.IsRegular() || entry.IsExecutable() } diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 36fd4a020b..ef3a45eabb 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -7,6 +7,7 @@ package setting import ( "path" "path/filepath" + "strings" ) // enumerates all the indexer queue types @@ -29,6 +30,8 @@ var ( IssueQueueDir string IssueQueueConnStr string IssueQueueBatchNumber int + FileExtensions map[string]bool + ExcludeExtensions bool }{ IssueType: "bleve", IssuePath: "indexers/issues.bleve", @@ -51,6 +54,9 @@ func newIndexerService() { if !filepath.IsAbs(Indexer.RepoPath) { Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath) } + Indexer.FileExtensions = extensionsFromString(sec.Key("REPO_INDEXER_EXTENSIONS").MustString("")) + Indexer.ExcludeExtensions = sec.Key("REPO_EXTENSIONS_LIST_EXCLUDE").MustBool(false) + Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType) @@ -58,3 +64,19 @@ func newIndexerService() { Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, "")) Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20) } + +func extensionsFromString(from string) map[string]bool { + extmap := make(map[string]bool) + for _, ext := range strings.Split(strings.ToLower(from), ",") { + ext = strings.TrimSpace(ext) + if ext == "." { + extmap[""] = true + } else if ext != "" && !strings.Contains(ext, ".") { + extmap[ext] = true + } + } + if len(extmap) == 0 { + return nil + } + return extmap +}