NYANDEV/forgejo
NYANDEV/forgejo
4
0
Fork 1
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-20 20:07:03 +01:00
Code Issues 2 Projects 1 Releases Packages Wiki Activity

Run processors on whole of text (#16155)

Browse source 
There is an inefficiency in the design of our processors which means that Emoji
and other processors run in order n^2 time.

This PR forces the processors to process the entirety of text node before passing
back up. The fundamental inefficiency remains but it should be significantly
ameliorated.

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2021-06-17 11:35:05 +01:00 committed by GitHub
parent 6ad5d0a306
commit 0db1048c3a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 414 additions and 316 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
modules/emoji/emoji.go
View file

@ -6,6 +6,7 @@
package emoji package emoji
import ( import (
"io"
"sort" "sort"
"strings" "strings"
"sync" "sync"
@ -145,6 +146,8 @@ func (n *rememberSecondWriteWriter) Write(p []byte) (int, error) {
if n.writecount == 2 { if n.writecount == 2 {
n.idx = n.pos n.idx = n.pos
n.end = n.pos + len(p) n.end = n.pos + len(p)
n.pos += len(p)
return len(p), io.EOF
} }
n.pos += len(p) n.pos += len(p)
return len(p), nil return len(p), nil
@ -155,6 +158,8 @@ func (n *rememberSecondWriteWriter) WriteString(s string) (int, error) {
if n.writecount == 2 { if n.writecount == 2 {
n.idx = n.pos n.idx = n.pos
n.end = n.pos + len(s) n.end = n.pos + len(s)
n.pos += len(s)
return len(s), io.EOF
} }
n.pos += len(s) n.pos += len(s)
return len(s), nil return len(s), nil

709
modules/markup/html.go
View file

@ -89,6 +89,7 @@ func isLinkStr(link string) bool {
return validLinksPattern.MatchString(link) return validLinksPattern.MatchString(link)
} }
// FIXME: This function is not concurrent safe
func getIssueFullPattern() *regexp.Regexp { func getIssueFullPattern() *regexp.Regexp {
if issueFullPattern == nil { if issueFullPattern == nil {
issueFullPattern = regexp.MustCompile(regexp.QuoteMeta(setting.AppURL) + issueFullPattern = regexp.MustCompile(regexp.QuoteMeta(setting.AppURL) +
@ -566,26 +567,38 @@ func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) {
} }
func mentionProcessor(ctx *RenderContext, node *html.Node) { func mentionProcessor(ctx *RenderContext, node *html.Node) {
// We replace only the first mention; other mentions will be addressed later start := 0
found, loc := references.FindFirstMentionBytes([]byte(node.Data)) next := node.NextSibling
if !found { for node != nil && node != next && start < len(node.Data) {
return // We replace only the first mention; other mentions will be addressed later
} found, loc := references.FindFirstMentionBytes([]byte(node.Data[start:]))
mention := node.Data[loc.Start:loc.End] if !found {
var teams string return
teams, ok := ctx.Metas["teams"]
// FIXME: util.URLJoin may not be necessary here:
// - setting.AppURL is defined to have a terminal '/' so unless mention[1:]
// is an AppSubURL link we can probably fallback to concatenation.
// team mention should follow @orgName/teamName style
if ok && strings.Contains(mention, "/") {
mentionOrgAndTeam := strings.Split(mention, "/")
if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
} }
return loc.Start += start
loc.End += start
mention := node.Data[loc.Start:loc.End]
var teams string
teams, ok := ctx.Metas["teams"]
// FIXME: util.URLJoin may not be necessary here:
// - setting.AppURL is defined to have a terminal '/' so unless mention[1:]
// is an AppSubURL link we can probably fallback to concatenation.
// team mention should follow @orgName/teamName style
if ok && strings.Contains(mention, "/") {
mentionOrgAndTeam := strings.Split(mention, "/")
if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
node = node.NextSibling.NextSibling
start = 0
continue
}
start = loc.End
continue
}
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
node = node.NextSibling.NextSibling
start = 0
} }
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
} }
func shortLinkProcessor(ctx *RenderContext, node *html.Node) { func shortLinkProcessor(ctx *RenderContext, node *html.Node) {
@ -593,188 +606,196 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) {
} }
func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) { func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) {
m := shortLinkPattern.FindStringSubmatchIndex(node.Data) next := node.NextSibling
if m == nil { for node != nil && node != next {
return m := shortLinkPattern.FindStringSubmatchIndex(node.Data)
} if m == nil {
return
}
content := node.Data[m[2]:m[3]] content := node.Data[m[2]:m[3]]
tail := node.Data[m[4]:m[5]] tail := node.Data[m[4]:m[5]]
props := make(map[string]string) props := make(map[string]string)
// MediaWiki uses [[link|text]], while GitHub uses [[text|link]] // MediaWiki uses [[link|text]], while GitHub uses [[text|link]]
// It makes page handling terrible, but we prefer GitHub syntax // It makes page handling terrible, but we prefer GitHub syntax
// And fall back to MediaWiki only when it is obvious from the look // And fall back to MediaWiki only when it is obvious from the look
// Of text and link contents // Of text and link contents
sl := strings.Split(content, "|") sl := strings.Split(content, "|")
for _, v := range sl { for _, v := range sl {
if equalPos := strings.IndexByte(v, '='); equalPos == -1 { if equalPos := strings.IndexByte(v, '='); equalPos == -1 {
// There is no equal in this argument; this is a mandatory arg // There is no equal in this argument; this is a mandatory arg
if props["name"] == "" { if props["name"] == "" {
if isLinkStr(v) { if isLinkStr(v) {
// If we clearly see it is a link, we save it so // If we clearly see it is a link, we save it so
// But first we need to ensure, that if both mandatory args provided // But first we need to ensure, that if both mandatory args provided
// look like links, we stick to GitHub syntax // look like links, we stick to GitHub syntax
if props["link"] != "" { if props["link"] != "" {
props["name"] = props["link"] props["name"] = props["link"]
}
props["link"] = strings.TrimSpace(v)
} else {
props["name"] = v
} }
props["link"] = strings.TrimSpace(v)
} else { } else {
props["name"] = v props["link"] = strings.TrimSpace(v)
} }
} else { } else {
props["link"] = strings.TrimSpace(v) // There is an equal; optional argument.
}
} else {
// There is an equal; optional argument.
sep := strings.IndexByte(v, '=') sep := strings.IndexByte(v, '=')
key, val := v[:sep], html.UnescapeString(v[sep+1:]) key, val := v[:sep], html.UnescapeString(v[sep+1:])
// When parsing HTML, x/net/html will change all quotes which are // When parsing HTML, x/net/html will change all quotes which are
// not used for syntax into UTF-8 quotes. So checking val[0] won't // not used for syntax into UTF-8 quotes. So checking val[0] won't
// be enough, since that only checks a single byte. // be enough, since that only checks a single byte.
if len(val) > 1 { if len(val) > 1 {
if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) || if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) ||
(strings.HasPrefix(val, "‘") && strings.HasSuffix(val, "’")) { (strings.HasPrefix(val, "‘") && strings.HasSuffix(val, "’")) {
const lenQuote = len("‘") const lenQuote = len("‘")
val = val[lenQuote : len(val)-lenQuote] val = val[lenQuote : len(val)-lenQuote]
} else if (strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"")) || } else if (strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"")) ||
(strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'")) { (strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'")) {
val = val[1 : len(val)-1] val = val[1 : len(val)-1]
} else if strings.HasPrefix(val, "'") && strings.HasSuffix(val, "’") { } else if strings.HasPrefix(val, "'") && strings.HasSuffix(val, "’") {
const lenQuote = len("‘") const lenQuote = len("‘")
val = val[1 : len(val)-lenQuote] val = val[1 : len(val)-lenQuote]
}
} }
props[key] = val
} }
props[key] = val
} }
}
var name, link string var name, link string
if props["link"] != "" { if props["link"] != "" {
link = props["link"] link = props["link"]
} else if props["name"] != "" { } else if props["name"] != "" {
link = props["name"] link = props["name"]
} }
if props["title"] != "" { if props["title"] != "" {
name = props["title"] name = props["title"]
} else if props["name"] != "" { } else if props["name"] != "" {
name = props["name"] name = props["name"]
} else {
name = link
}
name += tail
image := false
switch ext := filepath.Ext(link); ext {
// fast path: empty string, ignore
case "":
break
case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ".ico", ".svg":
image = true
}
childNode := &html.Node{}
linkNode := &html.Node{
FirstChild: childNode,
LastChild: childNode,
Type: html.ElementNode,
Data: "a",
DataAtom: atom.A,
}
childNode.Parent = linkNode
absoluteLink := isLinkStr(link)
if !absoluteLink {
if image {
link = strings.ReplaceAll(link, " ", "+")
} else { } else {
link = strings.ReplaceAll(link, " ", "-") name = link
}
if !strings.Contains(link, "/") {
link = url.PathEscape(link)
}
}
urlPrefix := ctx.URLPrefix
if image {
if !absoluteLink {
if IsSameDomain(urlPrefix) {
urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
}
if ctx.IsWiki {
link = util.URLJoin("wiki", "raw", link)
}
link = util.URLJoin(urlPrefix, link)
}
title := props["title"]
if title == "" {
title = props["alt"]
}
if title == "" {
title = path.Base(name)
}
alt := props["alt"]
if alt == "" {
alt = name
} }
// make the childNode an image - if we can, we also place the alt name += tail
childNode.Type = html.ElementNode image := false
childNode.Data = "img" switch ext := filepath.Ext(link); ext {
childNode.DataAtom = atom.Img // fast path: empty string, ignore
childNode.Attr = []html.Attribute{ case "":
{Key: "src", Val: link}, // leave image as false
{Key: "title", Val: title}, case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ".ico", ".svg":
{Key: "alt", Val: alt}, image = true
} }
if alt == "" {
childNode.Attr = childNode.Attr[:2] childNode := &html.Node{}
linkNode := &html.Node{
FirstChild: childNode,
LastChild: childNode,
Type: html.ElementNode,
Data: "a",
DataAtom: atom.A,
} }
} else { childNode.Parent = linkNode
absoluteLink := isLinkStr(link)
if !absoluteLink { if !absoluteLink {
if ctx.IsWiki { if image {
link = util.URLJoin("wiki", link) link = strings.ReplaceAll(link, " ", "+")
} else {
link = strings.ReplaceAll(link, " ", "-")
}
if !strings.Contains(link, "/") {
link = url.PathEscape(link)
} }
link = util.URLJoin(urlPrefix, link)
} }
childNode.Type = html.TextNode urlPrefix := ctx.URLPrefix
childNode.Data = name if image {
if !absoluteLink {
if IsSameDomain(urlPrefix) {
urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
}
if ctx.IsWiki {
link = util.URLJoin("wiki", "raw", link)
}
link = util.URLJoin(urlPrefix, link)
}
title := props["title"]
if title == "" {
title = props["alt"]
}
if title == "" {
title = path.Base(name)
}
alt := props["alt"]
if alt == "" {
alt = name
}
// make the childNode an image - if we can, we also place the alt
childNode.Type = html.ElementNode
childNode.Data = "img"
childNode.DataAtom = atom.Img
childNode.Attr = []html.Attribute{
{Key: "src", Val: link},
{Key: "title", Val: title},
{Key: "alt", Val: alt},
}
if alt == "" {
childNode.Attr = childNode.Attr[:2]
}
} else {
if !absoluteLink {
if ctx.IsWiki {
link = util.URLJoin("wiki", link)
}
link = util.URLJoin(urlPrefix, link)
}
childNode.Type = html.TextNode
childNode.Data = name
}
if noLink {
linkNode = childNode
} else {
linkNode.Attr = []html.Attribute{{Key: "href", Val: link}}
}
replaceContent(node, m[0], m[1], linkNode)
node = node.NextSibling.NextSibling
} }
if noLink {
linkNode = childNode
} else {
linkNode.Attr = []html.Attribute{{Key: "href", Val: link}}
}
replaceContent(node, m[0], m[1], linkNode)
} }
func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) { func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil { if ctx.Metas == nil {
return return
} }
m := getIssueFullPattern().FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
link := node.Data[m[0]:m[1]]
id := "#" + node.Data[m[2]:m[3]]
// extract repo and org name from matched link like next := node.NextSibling
// http://localhost:3000/gituser/myrepo/issues/1 for node != nil && node != next {
linkParts := strings.Split(path.Clean(link), "/") m := getIssueFullPattern().FindStringSubmatchIndex(node.Data)
matchOrg := linkParts[len(linkParts)-4] if m == nil {
matchRepo := linkParts[len(linkParts)-3] return
}
link := node.Data[m[0]:m[1]]
id := "#" + node.Data[m[2]:m[3]]
if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] { // extract repo and org name from matched link like
// TODO if m[4]:m[5] is not nil, then link is to a comment, // http://localhost:3000/gituser/myrepo/issues/1
// and we should indicate that in the text somehow linkParts := strings.Split(path.Clean(link), "/")
replaceContent(node, m[0], m[1], createLink(link, id, "ref-issue")) matchOrg := linkParts[len(linkParts)-4]
matchRepo := linkParts[len(linkParts)-3]
} else { if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] {
orgRepoID := matchOrg + "/" + matchRepo + id // TODO if m[4]:m[5] is not nil, then link is to a comment,
replaceContent(node, m[0], m[1], createLink(link, orgRepoID, "ref-issue")) // and we should indicate that in the text somehow
replaceContent(node, m[0], m[1], createLink(link, id, "ref-issue"))
} else {
orgRepoID := matchOrg + "/" + matchRepo + id
replaceContent(node, m[0], m[1], createLink(link, orgRepoID, "ref-issue"))
}
node = node.NextSibling.NextSibling
} }
} }
@ -782,70 +803,74 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil { if ctx.Metas == nil {
return return
} }
var ( var (
found bool found bool
ref *references.RenderizableReference ref *references.RenderizableReference
) )
_, exttrack := ctx.Metas["format"] next := node.NextSibling
alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric for node != nil && node != next {
_, exttrack := ctx.Metas["format"]
alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric
// Repos with external issue trackers might still need to reference local PRs // Repos with external issue trackers might still need to reference local PRs
// We need to concern with the first one that shows up in the text, whichever it is // We need to concern with the first one that shows up in the text, whichever it is
found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum) found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum)
if exttrack && alphanum { if exttrack && alphanum {
if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 { if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 {
if !found || ref2.RefLocation.Start < ref.RefLocation.Start { if !found || ref2.RefLocation.Start < ref.RefLocation.Start {
found = true found = true
ref = ref2 ref = ref2
}
} }
} }
} if !found {
if !found { return
return
}
var link *html.Node
reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
if exttrack && !ref.IsPull {
ctx.Metas["index"] = ref.Issue
link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue")
} else {
// Path determines the type of link that will be rendered. It's unknown at this point whether
// the linked item is actually a PR or an issue. Luckily it's of no real consequence because
// Gitea will redirect on click as appropriate.
path := "issues"
if ref.IsPull {
path = "pulls"
} }
if ref.Owner == "" {
link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue") var link *html.Node
reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
if exttrack && !ref.IsPull {
ctx.Metas["index"] = ref.Issue
link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue")
} else { } else {
link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue") // Path determines the type of link that will be rendered. It's unknown at this point whether
// the linked item is actually a PR or an issue. Luckily it's of no real consequence because
// Gitea will redirect on click as appropriate.
path := "issues"
if ref.IsPull {
path = "pulls"
}
if ref.Owner == "" {
link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue")
} else {
link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue")
}
} }
}
if ref.Action == references.XRefActionNone { if ref.Action == references.XRefActionNone {
replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link)
return node = node.NextSibling.NextSibling
} continue
}
// Decorate action keywords if actionable // Decorate action keywords if actionable
var keyword *html.Node var keyword *html.Node
if references.IsXrefActionable(ref, exttrack, alphanum) { if references.IsXrefActionable(ref, exttrack, alphanum) {
keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End]) keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End])
} else { } else {
keyword = &html.Node{ keyword = &html.Node{
Type: html.TextNode,
Data: node.Data[ref.ActionLocation.Start:ref.ActionLocation.End],
}
}
spaces := &html.Node{
Type: html.TextNode, Type: html.TextNode,
Data: node.Data[ref.ActionLocation.Start:ref.ActionLocation.End], Data: node.Data[ref.ActionLocation.End:ref.RefLocation.Start],
} }
replaceContentList(node, ref.ActionLocation.Start, ref.RefLocation.End, []*html.Node{keyword, spaces, link})
node = node.NextSibling.NextSibling.NextSibling.NextSibling
} }
spaces := &html.Node{
Type: html.TextNode,
Data: node.Data[ref.ActionLocation.End:ref.RefLocation.Start],
}
replaceContentList(node, ref.ActionLocation.Start, ref.RefLocation.End, []*html.Node{keyword, spaces, link})
} }
// fullSha1PatternProcessor renders SHA containing URLs // fullSha1PatternProcessor renders SHA containing URLs
@ -853,86 +878,112 @@ func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil { if ctx.Metas == nil {
return return
} }
m := anySHA1Pattern.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
urlFull := node.Data[m[0]:m[1]] next := node.NextSibling
text := base.ShortSha(node.Data[m[2]:m[3]]) for node != nil && node != next {
m := anySHA1Pattern.FindStringSubmatchIndex(node.Data)
// 3rd capture group matches a optional path if m == nil {
subpath := "" return
if m[5] > 0 {
subpath = node.Data[m[4]:m[5]]
}
// 4th capture group matches a optional url hash
hash := ""
if m[7] > 0 {
hash = node.Data[m[6]:m[7]][1:]
}
start := m[0]
end := m[1]
// If url ends in '.', it's very likely that it is not part of the
// actual url but used to finish a sentence.
if strings.HasSuffix(urlFull, ".") {
end--
urlFull = urlFull[:len(urlFull)-1]
if hash != "" {
hash = hash[:len(hash)-1]
} else if subpath != "" {
subpath = subpath[:len(subpath)-1]
} }
}
if subpath != "" { urlFull := node.Data[m[0]:m[1]]
text += subpath text := base.ShortSha(node.Data[m[2]:m[3]])
}
if hash != "" { // 3rd capture group matches a optional path
text += " (" + hash + ")" subpath := ""
} if m[5] > 0 {
subpath = node.Data[m[4]:m[5]]
}
replaceContent(node, start, end, createCodeLink(urlFull, text, "commit")) // 4th capture group matches a optional url hash
hash := ""
if m[7] > 0 {
hash = node.Data[m[6]:m[7]][1:]
}
start := m[0]
end := m[1]
// If url ends in '.', it's very likely that it is not part of the
// actual url but used to finish a sentence.
if strings.HasSuffix(urlFull, ".") {
end--
urlFull = urlFull[:len(urlFull)-1]
if hash != "" {
hash = hash[:len(hash)-1]
} else if subpath != "" {
subpath = subpath[:len(subpath)-1]
}
}
if subpath != "" {
text += subpath
}
if hash != "" {
text += " (" + hash + ")"
}
replaceContent(node, start, end, createCodeLink(urlFull, text, "commit"))
node = node.NextSibling.NextSibling
}
} }
// emojiShortCodeProcessor for rendering text like :smile: into emoji // emojiShortCodeProcessor for rendering text like :smile: into emoji
func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) { func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) {
m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data) start := 0
if m == nil { next := node.NextSibling
return for node != nil && node != next && start < len(node.Data) {
} m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data[start:])
if m == nil {
alias := node.Data[m[0]:m[1]]
alias = strings.ReplaceAll(alias, ":", "")
converted := emoji.FromAlias(alias)
if converted == nil {
// check if this is a custom reaction
s := strings.Join(setting.UI.Reactions, " ") + "gitea"
if strings.Contains(s, alias) {
replaceContent(node, m[0], m[1], createCustomEmoji(alias, "emoji"))
return return
} }
return m[0] += start
} m[1] += start
replaceContent(node, m[0], m[1], createEmoji(converted.Emoji, "emoji", converted.Description)) start = m[1]
alias := node.Data[m[0]:m[1]]
alias = strings.ReplaceAll(alias, ":", "")
converted := emoji.FromAlias(alias)
if converted == nil {
// check if this is a custom reaction
s := strings.Join(setting.UI.Reactions, " ") + "gitea"
if strings.Contains(s, alias) {
replaceContent(node, m[0], m[1], createCustomEmoji(alias, "emoji"))
node = node.NextSibling.NextSibling
start = 0
continue
}
continue
}
replaceContent(node, m[0], m[1], createEmoji(converted.Emoji, "emoji", converted.Description))
node = node.NextSibling.NextSibling
start = 0
}
} }
// emoji processor to match emoji and add emoji class // emoji processor to match emoji and add emoji class
func emojiProcessor(ctx *RenderContext, node *html.Node) { func emojiProcessor(ctx *RenderContext, node *html.Node) {
m := emoji.FindEmojiSubmatchIndex(node.Data) start := 0
if m == nil { next := node.NextSibling
return for node != nil && node != next && start < len(node.Data) {
} m := emoji.FindEmojiSubmatchIndex(node.Data[start:])
if m == nil {
return
}
m[0] += start
m[1] += start
codepoint := node.Data[m[0]:m[1]] codepoint := node.Data[m[0]:m[1]]
val := emoji.FromCode(codepoint) start = m[1]
if val != nil { val := emoji.FromCode(codepoint)
replaceContent(node, m[0], m[1], createEmoji(codepoint, "emoji", val.Description)) if val != nil {
replaceContent(node, m[0], m[1], createEmoji(codepoint, "emoji", val.Description))
node = node.NextSibling.NextSibling
start = 0
}
} }
} }
@ -942,49 +993,70 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" { if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" {
return return
} }
m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
hash := node.Data[m[2]:m[3]]
// The regex does not lie, it matches the hash pattern.
// However, a regex cannot know if a hash actually exists or not.
// We could assume that a SHA1 hash should probably contain alphas AND numerics
// but that is not always the case.
// Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash
// as used by git and github for linking and thus we have to do similar.
// Because of this, we check to make sure that a matched hash is actually
// a commit in the repository before making it a link.
if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil {
if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
}
return
}
replaceContent(node, m[2], m[3], start := 0
createCodeLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash), base.ShortSha(hash), "commit")) next := node.NextSibling
for node != nil && node != next && start < len(node.Data) {
m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data[start:])
if m == nil {
return
}
m[2] += start
m[3] += start
hash := node.Data[m[2]:m[3]]
// The regex does not lie, it matches the hash pattern.
// However, a regex cannot know if a hash actually exists or not.
// We could assume that a SHA1 hash should probably contain alphas AND numerics
// but that is not always the case.
// Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash
// as used by git and github for linking and thus we have to do similar.
// Because of this, we check to make sure that a matched hash is actually
// a commit in the repository before making it a link.
if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil {
if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
}
start = m[3]
continue
}
replaceContent(node, m[2], m[3],
createCodeLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash), base.ShortSha(hash), "commit"))
start = 0
node = node.NextSibling.NextSibling
}
} }
// emailAddressProcessor replaces raw email addresses with a mailto: link. // emailAddressProcessor replaces raw email addresses with a mailto: link.
func emailAddressProcessor(ctx *RenderContext, node *html.Node) { func emailAddressProcessor(ctx *RenderContext, node *html.Node) {
m := emailRegex.FindStringSubmatchIndex(node.Data) next := node.NextSibling
if m == nil { for node != nil && node != next {
return m := emailRegex.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
mail := node.Data[m[2]:m[3]]
replaceContent(node, m[2], m[3], createLink("mailto:"+mail, mail, "mailto"))
node = node.NextSibling.NextSibling
} }
mail := node.Data[m[2]:m[3]]
replaceContent(node, m[2], m[3], createLink("mailto:"+mail, mail, "mailto"))
} }
// linkProcessor creates links for any HTTP or HTTPS URL not captured by // linkProcessor creates links for any HTTP or HTTPS URL not captured by
// markdown. // markdown.
func linkProcessor(ctx *RenderContext, node *html.Node) { func linkProcessor(ctx *RenderContext, node *html.Node) {
m := common.LinkRegex.FindStringIndex(node.Data) next := node.NextSibling
if m == nil { for node != nil && node != next {
return m := common.LinkRegex.FindStringIndex(node.Data)
if m == nil {
return
}
uri := node.Data[m[0]:m[1]]
replaceContent(node, m[0], m[1], createLink(uri, uri, "link"))
node = node.NextSibling.NextSibling
} }
uri := node.Data[m[0]:m[1]]
replaceContent(node, m[0], m[1], createLink(uri, uri, "link"))
} }
func genDefaultLinkProcessor(defaultLink string) processor { func genDefaultLinkProcessor(defaultLink string) processor {
@ -1008,12 +1080,17 @@ func genDefaultLinkProcessor(defaultLink string) processor {
// descriptionLinkProcessor creates links for DescriptionHTML // descriptionLinkProcessor creates links for DescriptionHTML
func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) { func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) {
m := common.LinkRegex.FindStringIndex(node.Data) next := node.NextSibling
if m == nil { for node != nil && node != next {
return m := common.LinkRegex.FindStringIndex(node.Data)
if m == nil {
return
}
uri := node.Data[m[0]:m[1]]
replaceContent(node, m[0], m[1], createDescriptionLink(uri, uri))
node = node.NextSibling.NextSibling
} }
uri := node.Data[m[0]:m[1]]
replaceContent(node, m[0], m[1], createDescriptionLink(uri, uri))
} }
func createDescriptionLink(href, content string) *html.Node { func createDescriptionLink(href, content string) *html.Node {

16
modules/markup/html_test.go
View file

@ -464,3 +464,19 @@ func TestIssue16020(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, data, res.String()) assert.Equal(t, data, res.String())
} }
func BenchmarkEmojiPostprocess(b *testing.B) {
data := "🥰 "
for len(data) < 1<<16 {
data += data
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
var res strings.Builder
err := PostProcess(&RenderContext{
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
assert.NoError(b, err)
}
}