Correctly support linguist-documentation=false

If a documentation file is marked with a `linguist-documentation=false`
attribute, include it in language stats.

However, make sure that we do *not* include documentation languages as
fallback.

Added a new test case to exercise the formerly buggy behaviour.

Problem discovered while reviewing @KN4CK3R's tests from gitea#29267.

Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>

modules/git/repo_attribute.go

@@ -336,10 +336,3 @@ func attributeToBool(attr map[string]string, name string) optional.Option[bool]
 	}
 	return optional.None[bool]()
 }
-
-func attributeToString(attr map[string]string, name string) optional.Option[string] {
-	if value, has := attr[name]; has && value != "unspecified" {
-		return optional.Some(value)
-	}
-	return optional.None[string]()
-}

modules/git/repo_language_stats_gogit.go

@@ -138,21 +138,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 		}
 
 		included, checked := includedLanguage[language]
+		langType := enry.GetLanguageType(language)
 		if !checked {
-			langtype := enry.GetLanguageType(language)
-			included = langtype == enry.Programming || langtype == enry.Markup
-			if !included {
-				if isTrue(isDetectable) {
-					included = true
-				} else {
-					return nil
-				}
+			included = langType == enry.Programming || langType == enry.Markup
+			if !included && (isTrue(isDetectable) || (langType == enry.Prose && isFalse(isDocumentation))) {
+				included = true
 			}
 			includedLanguage[language] = included
 		}
 		if included {
 			sizes[language] += f.Size
 		} else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) {
+			// Only consider Programming or Markup languages as fallback
+			if !(langType == enry.Programming || langType == enry.Markup) {
+				return nil
+			}
+
 			firstExcludedLanguage = language
 			firstExcludedLanguageSize += f.Size
 		}

modules/git/repo_language_stats_nogogit.go

@@ -197,25 +197,24 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 		}
 
 		included, checked := includedLanguage[language]
+		langType := enry.GetLanguageType(language)
 		if !checked {
-			langType := enry.GetLanguageType(language)
 			included = langType == enry.Programming || langType == enry.Markup
-			if !included {
-				if isTrue(isDetectable) {
-					included = true
-				} else {
-					continue
-				}
+			if !included && (isTrue(isDetectable) || (langType == enry.Prose && isFalse(isDocumentation))) {
+				included = true
 			}
 			includedLanguage[language] = included
 		}
 		if included {
 			sizes[language] += f.Size()
 		} else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) {
+			// Only consider Programming or Markup languages as fallback
+			if !(langType == enry.Programming || langType == enry.Markup) {
+				continue
+			}
 			firstExcludedLanguage = language
 			firstExcludedLanguageSize += f.Size()
 		}
-		continue
 	}
 
 	// If there are no included languages add the first excluded language

tests/integration/linguist_test.go

@@ -251,5 +251,18 @@ func TestLinguistSupport(t *testing.T) {
 				assertFileLanguage(t, "/blame/branch/main/foo.c", "Bash")
 			})
 		})
+
+		// 10. Marking a file as non-documentation
+		t.Run("linguist-documentation=false", func(t *testing.T) {
+			defer tests.PrintCurrentTest(t)()
+
+			repo, sha, f := prep(t, "README.md linguist-documentation=false\n")
+			defer f()
+
+			langs := getFreshLanguageStats(t, repo, sha)
+			assert.Len(t, langs, 2)
+			assert.Equal(t, "Markdown", langs[0].Language)
+			assert.Equal(t, "C", langs[1].Language)
+		})
 	})
 }