feat(search): Support Magik language file filter in search (#63110)

Adds support to the lang file filter for Magik language (`lang: Magik`).
In order to do that we add wrappers around go-enry and update search
code to use them. This provides flexibility for us in the future to
support other languages that are not in Linguist as well.

Author: mmanela

client/common/src/languages.ts

@@ -46,6 +46,20 @@ export const POPULAR_LANGUAGES: string[] = [
     'Zig',
 ]
 
+/**
+ * Languages that are not present in the go-enry library
+ * and so are not in go-enry either
+ */
+const LANGUAGES_NOT_IN_ENRY = [
+    // Not in Linguist and they are not likely to add
+    'Magik',
+
+    // Add to linguist on 6/7/24
+    // can remove once go-enry package updates
+    // to that linguist version
+    'Pkl',
+]
+
 /**
  * A list of all supported languages, ranking the popular languages higher by
  * including them first.
@@ -721,4 +735,6 @@ export const ALL_LANGUAGES = uniq(
         'wisp',
         'xBase',
     ])
+        .concat(LANGUAGES_NOT_IN_ENRY)
+        .sort()
 )

internal/search/job/jobutil/BUILD.bazel

@@ -57,10 +57,10 @@ go_library(
         "//internal/telemetry/telemetryrecorder",
         "//internal/trace",
         "//internal/usagestats",
+        "//lib/codeintel/languages",
         "//lib/errors",
         "//lib/iterator",
         "//schema",
-        "@com_github_go_enry_go_enry_v2//:go-enry",
         "@com_github_grafana_regexp//:regexp",
         "@com_github_sourcegraph_conc//pool",
         "@com_github_sourcegraph_log//:log",

internal/search/job/jobutil/job.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-enry/go-enry/v2"
 	"github.com/grafana/regexp"
 
 	zoektquery "github.com/sourcegraph/zoekt/query"
@@ -28,6 +27,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/search/structural"
 	"github.com/sourcegraph/sourcegraph/internal/search/zoekt"
 	"github.com/sourcegraph/sourcegraph/internal/searcher/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/languages"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 	"github.com/sourcegraph/sourcegraph/schema"
 )
@@ -722,7 +722,7 @@ func toTextPatternInfo(b query.Basic, resultTypes result.Types, feat *search.Fea
 func toLangFilters(aliases []string) []string {
 	var filters []string
 	for _, alias := range aliases {
-		lang, _ := enry.GetLanguageByAlias(alias) // Invariant: lang is valid.
+		lang, _ := languages.GetLanguageByAlias(alias) // Invariant: lang is valid.
 		if !slices.Contains(filters, lang) {
 			filters = append(filters, lang)
 		}

internal/search/job/jobutil/job_test.go

@@ -1221,6 +1221,54 @@ func TestNewPlanJob(t *testing.T) {
                 (repoNamePatterns . [(?i)foo bar baz])))
             REPOSCOMPUTEEXCLUDED
             NOOP))))))
+`),
+		},
+
+		// This test shows that we can handle languages not in Linguist
+		{
+			query:      `context:global repo:sourcegraph/.* lang:magik`,
+			protocol:   search.Streaming,
+			searchType: query.SearchTypeKeyword,
+			want: autogold.Expect(`
+(LOG
+  (ALERT
+    (features . error decoding features)
+    (protocol . Streaming)
+    (onSourcegraphDotCom . true)
+    (query . )
+    (originalQuery . )
+    (patternType . keyword)
+    (TIMEOUT
+      (timeout . 20s)
+      (LIMIT
+        (limit . 10000)
+        (PARALLEL
+          (REPOPAGER
+            (containsRefGlobs . false)
+            (repoOpts.repoFilters . [sourcegraph/.*])
+            (repoOpts.searchContextSpec . global)
+            (PARTIALREPOS
+              (ZOEKTREPOSUBSETTEXTSEARCH
+                (fileMatchLimit . 10000)
+                (select . )
+                (zoektQueryRegexps . [(?i)(?im:\.MAGIK$)])
+                (query . file_regex:"(?i:\\.MAGIK)(?m:$)")
+                (type . text))))
+          (REPOPAGER
+            (containsRefGlobs . false)
+            (repoOpts.repoFilters . [sourcegraph/.*])
+            (repoOpts.searchContextSpec . global)
+            (PARTIALREPOS
+              (SEARCHERTEXTSEARCH
+                (useFullDeadline . true)
+                (patternInfo . TextPatternInfo{//,filematchlimit:10000,lang:magik,f:"(?i)\\.magik$"})
+                (numRepos . 0)
+                (pathRegexps . [(?i)\.magik$])
+                (indexed . false))))
+          (REPOSCOMPUTEEXCLUDED
+            (repoOpts.repoFilters . [sourcegraph/.*])
+            (repoOpts.searchContextSpec . global))
+          NOOP)))))
 `),
 		},
 	}
@@ -1278,6 +1326,7 @@ func TestToTextPatternInfo(t *testing.T) {
 	cases := []struct {
 		input  string
 		output autogold.Value
+		feat   search.Features
 	}{{
 		input:  `type:repo archived`,
 		output: autogold.Expect(`{"Query":{"Value":"archived","IsNegated":false,"IsRegExp":false},"IsStructuralPat":false,"CombyRule":"","IsCaseSensitive":false,"FileMatchLimit":30,"Index":"yes","Select":[],"IncludePaths":null,"ExcludePaths":"","IncludeLangs":null,"ExcludeLangs":null,"PathPatternsAreCaseSensitive":false,"PatternMatchesContent":false,"PatternMatchesPath":false,"Languages":null}`),
@@ -1410,9 +1459,17 @@ func TestToTextPatternInfo(t *testing.T) {
 	}, {
 		input:  `repo:^github\.com/sgtest/sourcegraph-typescript$ file:^README\.md "basic :[_] access :[_]" patterntype:structural`,
 		output: autogold.Expect(`{"Query":{"Value":"\"basic :[_] access :[_]\"","IsNegated":false,"IsRegExp":false},"IsStructuralPat":true,"CombyRule":"","IsCaseSensitive":false,"FileMatchLimit":30,"Index":"yes","Select":[],"IncludePaths":["^README\\.md"],"ExcludePaths":"","IncludeLangs":null,"ExcludeLangs":null,"PathPatternsAreCaseSensitive":false,"PatternMatchesContent":true,"PatternMatchesPath":true,"Languages":null}`),
+	}, {
+		input:  `sgtest lang:magik type:file`,
+		feat:   search.Features{ContentBasedLangFilters: true},
+		output: autogold.Expect(`{"Query":{"Value":"sgtest","IsNegated":false,"IsRegExp":false},"IsStructuralPat":false,"CombyRule":"","IsCaseSensitive":false,"FileMatchLimit":30,"Index":"yes","Select":[],"IncludePaths":null,"ExcludePaths":"","IncludeLangs":["Magik"],"ExcludeLangs":null,"PathPatternsAreCaseSensitive":false,"PatternMatchesContent":true,"PatternMatchesPath":false,"Languages":["magik"]}`),
+	}, {
+		input:  `sgtest lang:magik type:file`,
+		feat:   search.Features{ContentBasedLangFilters: false},
+		output: autogold.Expect(`{"Query":{"Value":"sgtest","IsNegated":false,"IsRegExp":false},"IsStructuralPat":false,"CombyRule":"","IsCaseSensitive":false,"FileMatchLimit":30,"Index":"yes","Select":[],"IncludePaths":["(?i)\\.magik$"],"ExcludePaths":"","IncludeLangs":null,"ExcludeLangs":null,"PathPatternsAreCaseSensitive":false,"PatternMatchesContent":true,"PatternMatchesPath":false,"Languages":["magik"]}`),
 	}}
 
-	test := func(input string) string {
+	test := func(input string, feat search.Features) string {
 		searchType := overrideSearchType(input, query.SearchTypeLiteral)
 		plan, err := query.Pipeline(query.Init(input, searchType))
 		if err != nil {
@@ -1423,14 +1480,14 @@ func TestToTextPatternInfo(t *testing.T) {
 		}
 		b := plan[0]
 		resultTypes := computeResultTypes(b, query.SearchTypeLiteral, defaultResultTypes)
-		p := toTextPatternInfo(b, resultTypes, &search.Features{}, limits.DefaultMaxSearchResults)
+		p := toTextPatternInfo(b, resultTypes, &feat, limits.DefaultMaxSearchResults)
 		v, _ := json.Marshal(p)
 		return string(v)
 	}
 
 	for _, tc := range cases {
 		t.Run(tc.input, func(t *testing.T) {
-			tc.output.Equal(t, test(tc.input))
+			tc.output.Equal(t, test(tc.input, tc.feat))
 		})
 	}
 }

internal/search/query/BUILD.bazel

@@ -27,8 +27,8 @@ go_library(
         "//internal/gitserver/gitdomain",
         "//internal/search/filter",
         "//internal/search/limits",
+        "//lib/codeintel/languages",
         "//lib/errors",
-        "@com_github_go_enry_go_enry_v2//:go-enry",
         "@com_github_go_enry_go_enry_v2//data",
         "@com_github_grafana_regexp//:regexp",
         "@com_github_grafana_regexp//syntax",

internal/search/query/helpers.go

@@ -4,9 +4,10 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/go-enry/go-enry/v2"
 	"github.com/go-enry/go-enry/v2/data"
 	"github.com/grafana/regexp"
+
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/languages"
 )
 
 // UnionRegExps separates values with a | operator to create a string
@@ -57,8 +58,8 @@ var filenamesFromLanguage = func() map[string][]string {
 // LangToFileRegexp converts a lang: parameter to its corresponding file
 // patterns for file filters. The lang value must be valid, cf. validate.go
 func LangToFileRegexp(lang string) string {
-	lang, _ = enry.GetLanguageByAlias(lang) // Invariant: lang is valid.
-	extensions := enry.GetLanguageExtensions(lang)
+	lang, _ = languages.GetLanguageByAlias(lang) // Invariant: lang is valid.
+	extensions := languages.GetLanguageExtensions(lang)
 	patterns := make([]string, len(extensions))
 	for i, e := range extensions {
 		// Add `\.ext$` pattern to match files with the given extension.

internal/search/query/validate.go

@@ -5,11 +5,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-enry/go-enry/v2"
 	"github.com/grafana/regexp"
 
 	"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
 	"github.com/sourcegraph/sourcegraph/internal/search/filter"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/languages"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
@@ -199,7 +199,7 @@ func validateField(field, value string, negated bool, seen map[string]struct{})
 	}
 
 	isLanguage := func() error {
-		_, ok := enry.GetLanguageByAlias(value)
+		_, ok := languages.GetLanguageByAlias(value)
 		if !ok {
 			return errors.Errorf("unknown language: %q", value)
 		}

internal/search/zoekt/BUILD.bazel

@@ -31,8 +31,8 @@ go_library(
         "//internal/search/zoektquery",
         "//internal/trace",
         "//internal/types",
+        "//lib/codeintel/languages",
         "//lib/errors",
-        "@com_github_go_enry_go_enry_v2//:go-enry",
         "@com_github_grafana_regexp//:regexp",
         "@com_github_roaringbitmap_roaring//:roaring",
         "@com_github_sourcegraph_log//:log",

internal/search/zoekt/query.go

@@ -3,12 +3,11 @@ package zoekt
 import (
 	"regexp/syntax" //nolint:depguard // using the grafana fork of regexp clashes with zoekt, which uses the std regexp/syntax.
 
-	"github.com/go-enry/go-enry/v2"
-
 	"github.com/sourcegraph/sourcegraph/internal/search"
 	"github.com/sourcegraph/sourcegraph/internal/search/query"
 	"github.com/sourcegraph/sourcegraph/internal/search/result"
 	"github.com/sourcegraph/sourcegraph/internal/search/zoektquery"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/languages"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 
 	zoekt "github.com/sourcegraph/zoekt/query"
@@ -82,7 +81,7 @@ func QueryToZoektQuery(b query.Basic, resultTypes result.Types, feat *search.Fea
 }
 
 func toLangFilter(lang string) zoekt.Q {
-	lang, _ = enry.GetLanguageByAlias(lang) // Invariant: lang is valid.
+	lang, _ = languages.GetLanguageByAlias(lang) // Invariant: lang is valid.
 	return &zoekt.Language{Language: lang}
 }
 

lib/codeintel/languages/extensions.go

@@ -2,10 +2,32 @@ package languages
 
 import (
 	"path/filepath"
+	"strings"
 
 	"github.com/go-enry/go-enry/v2"
 )
 
+// getLanguagesByAlias is a replacement for enry.GetLanguagesByAlias
+// It supports languages that are missing in go-enry
+func GetLanguageByAlias(alias string) (lang string, ok bool) {
+	normalizedAlias := strings.ToLower(alias)
+	if lang, ok = unsupportedByEnryAliasMap[normalizedAlias]; ok {
+		return lang, true
+	}
+
+	return enry.GetLanguageByAlias(normalizedAlias)
+}
+
+// getLanguagesByExtension is a replacement for enry.GetLanguagesByExtension
+// It supports languages that are missing in go-enry
+func GetLanguageExtensions(alias string) []string {
+	if lang, ok := unsupportedByEnryNameToExtensionMap[alias]; ok {
+		return []string{lang}
+	}
+
+	return enry.GetLanguageExtensions(alias)
+}
+
 // getLanguagesByExtension is a replacement for enry.GetLanguagesByExtension
 // to work around the following limitations:
 //   - For some extensions which are overwhelmingly used by a certain file type
@@ -17,7 +39,7 @@ func getLanguagesByExtension(path string) (candidates []string, isLikelyBinaryFi
 	if ext == "" {
 		return nil, false
 	}
-	if lang, ok := unsupportedByEnryExtensionsMap[ext]; ok {
+	if lang, ok := unsupportedByEnryExtensionToNameMap[ext]; ok {
 		return []string{lang}, false
 	}
 	if _, ok := commonBinaryFileExtensions[ext[1:]]; ok {
@@ -62,13 +84,33 @@ var overrideAmbiguousExtensionsMap = map[string]string{
 	// for other variants of YAML, hence only 'YAML' is picked by enry.
 }
 
-var unsupportedByEnryExtensionsMap = map[string]string{
+var unsupportedByEnryExtensionToNameMap = map[string]string{
 	// Pkl Configuration Language (https://pkl-lang.org/)
+	// NOTE: Add to linguist on 6/7/24
+	// can remove once go-enry package updates
+	// to that linguist version
 	".pkl": "Pkl",
 	// Magik Language
 	".magik": "Magik",
 }
 
+var unsupportedByEnryNameToExtensionMap = reverseMap(unsupportedByEnryExtensionToNameMap)
+
+var unsupportedByEnryAliasMap = map[string]string{
+	// Pkl Configuration Language (https://pkl-lang.org/)
+	"pkl": "Pkl",
+	// Magik Language
+	"magik": "Magik",
+}
+
+func reverseMap(m map[string]string) map[string]string {
+	n := make(map[string]string, len(m))
+	for k, v := range m {
+		n[v] = k
+	}
+	return n
+}
+
 // Source: https://github.com/sindresorhus/binary-extensions/blob/main/binary-extensions.json
 // License: https://github.com/sindresorhus/binary-extensions/blob/main/license
 // Replace the contents with