Added the message parser

channel.go

@@ -133,7 +133,7 @@ func (ch *Channel) JoinServer(ctx context.Context, ct cchat.MessagesContainer) (
 		}))
 	} else {
 		constructor = func(m discord.Message) cchat.MessageCreate {
-			return NewDirectMessage(m)
+			return NewDirectMessage(m, ch.session)
 		}
 	}
 

go.mod

@@ -2,9 +2,14 @@ module github.com/diamondburned/cchat-discord
 
 go 1.14
 
+replace github.com/diamondburned/ningen => ../../ningen/
+
 require (
+	github.com/davecgh/go-spew v1.1.1
 	github.com/diamondburned/arikawa v0.9.4
-	github.com/diamondburned/cchat v0.0.28
-	github.com/diamondburned/ningen v0.0.0-20200610212436-159f7105a2be
+	github.com/diamondburned/cchat v0.0.31
+	github.com/diamondburned/ningen v0.0.0-20200618230530-16d4d7fbc521
+	github.com/go-test/deep v1.0.6
 	github.com/pkg/errors v0.9.1
+	github.com/yuin/goldmark v1.1.30
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/diamondburned/arikawa v0.8.7-0.20200522214036-530bff74a2c6/go.mod h1:nIhVIatzTQhPUa7NB8w4koG1RF9gYbpAr8Fj8sKq660=
 github.com/diamondburned/arikawa v0.9.4 h1:Mrp0Vz9R2afbvhWS6m/oLIQy22/uxXb459LUv7qrZPA=
 github.com/diamondburned/arikawa v0.9.4/go.mod h1:nIhVIatzTQhPUa7NB8w4koG1RF9gYbpAr8Fj8sKq660=
@@ -5,8 +7,12 @@ github.com/diamondburned/cchat v0.0.26 h1:QBt4d65uzUPJz3jF8b2pJ09Jz8LeBRyG2ol47F
 github.com/diamondburned/cchat v0.0.26/go.mod h1:+zXktogE45A0om4fT6B/z6Ii7FXNafjxsNspI0rlhbU=
 github.com/diamondburned/cchat v0.0.28 h1:+1VnltW0rl8/NZTUP+x89jVhi3YTTR+e6iLprZ7HcwM=
 github.com/diamondburned/cchat v0.0.28/go.mod h1:+zXktogE45A0om4fT6B/z6Ii7FXNafjxsNspI0rlhbU=
+github.com/diamondburned/cchat v0.0.31 h1:yUgrh5xbGX0R55glyxYtVewIDL2eXLJ+okIEfVaVoFk=
+github.com/diamondburned/cchat v0.0.31/go.mod h1:+zXktogE45A0om4fT6B/z6Ii7FXNafjxsNspI0rlhbU=
 github.com/diamondburned/ningen v0.0.0-20200610212436-159f7105a2be h1:mUw8X/YzJGFSdL8y3Q/XqyzqPyIMNVSDyZGOP3JXgJA=
 github.com/diamondburned/ningen v0.0.0-20200610212436-159f7105a2be/go.mod h1:B2hq2B4va1MlnMmXuv9vXmyu9gscxJLmwrmcSB1Les8=
+github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
+github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY=
 github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
@@ -15,6 +21,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
 github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
+github.com/yuin/goldmark v1.1.30 h1:j4d4Lw3zqZelDhBksEo3BnWg9xhXRQGJPPSL6OApZjI=
 github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

message.go

@@ -131,7 +131,7 @@ func NewMessageCreate(c *gateway.MessageCreateEvent, s *Session) Message {
 	// This should not error.
 	g, err := s.Store.Guild(c.GuildID)
 	if err != nil {
-		return NewMessage(c.Message, NewUser(c.Author))
+		return NewMessage(c.Message, s, NewUser(c.Author))
 	}
 
 	if c.Member == nil {
@@ -139,10 +139,10 @@ func NewMessageCreate(c *gateway.MessageCreateEvent, s *Session) Message {
 	}
 	if c.Member == nil {
 		s.Members.RequestMember(c.GuildID, c.Author.ID)
-		return NewMessage(c.Message, NewUser(c.Author))
+		return NewMessage(c.Message, s, NewUser(c.Author))
 	}
 
-	return NewMessage(c.Message, NewGuildMember(*c.Member, *g))
+	return NewMessage(c.Message, s, NewGuildMember(*c.Member, *g))
 }
 
 // NewBacklogMessage uses the session to create a message fetched from the
@@ -152,27 +152,27 @@ func NewBacklogMessage(m discord.Message, s *Session, g discord.Guild) Message {
 	// If the message doesn't have a guild, then we don't need all the
 	// complicated member fetching process.
 	if !m.GuildID.Valid() {
-		return NewMessage(m, NewUser(m.Author))
+		return NewMessage(m, s, NewUser(m.Author))
 	}
 
 	mem, err := s.Store.Member(m.GuildID, m.Author.ID)
 	if err != nil {
 		s.Members.RequestMember(m.GuildID, m.Author.ID)
-		return NewMessage(m, NewUser(m.Author))
+		return NewMessage(m, s, NewUser(m.Author))
 	}
 
-	return NewMessage(m, NewGuildMember(*mem, g))
+	return NewMessage(m, s, NewGuildMember(*mem, g))
 }
 
-func NewDirectMessage(m discord.Message) Message {
-	return NewMessage(m, NewUser(m.Author))
+func NewDirectMessage(m discord.Message, s *Session) Message {
+	return NewMessage(m, s, NewUser(m.Author))
 }
 
-func NewMessage(m discord.Message, author Author) Message {
+func NewMessage(m discord.Message, s *Session, author Author) Message {
 	return Message{
 		messageHeader: newHeader(m),
 		author:        author,
-		content:       text.Rich{Content: m.Content},
+		content:       segments.ParseMessage(&m, s.Store),
 	}
 }
 

segments/blockquote.go

@@ -0,0 +1,45 @@
+package segments
+
+import (
+	"github.com/diamondburned/cchat/text"
+	"github.com/yuin/goldmark/ast"
+)
+
+type BlockquoteSegment struct {
+	start, end int
+}
+
+var _ text.Quoteblocker = (*BlockquoteSegment)(nil)
+
+func (r *TextRenderer) blockquote(n *ast.Blockquote, enter bool) ast.WalkStatus {
+	if enter {
+		// Create a segment.
+		var seg = BlockquoteSegment{start: r.i()}
+
+		// A blockquote contains a paragraph each line. Because Discord.
+		for child := n.FirstChild(); child != nil; child = child.NextSibling() {
+			r.buf.WriteString("> ")
+
+			ast.Walk(child, func(node ast.Node, enter bool) (ast.WalkStatus, error) {
+				// We only call when entering, since we don't want to trigger a
+				// hard new line after each paragraph.
+				if enter {
+					return r.renderNode(node, enter)
+				}
+				return ast.WalkContinue, nil
+			})
+		}
+
+		// Write the end of the segment.
+		seg.end = r.i()
+		r.append(seg)
+	}
+
+	return ast.WalkSkipChildren
+}
+
+func (b BlockquoteSegment) Bounds() (start, end int) {
+	return b.start, b.end
+}
+
+func (b BlockquoteSegment) Quote() {}

segments/codeblock.go

@@ -0,0 +1,45 @@
+package segments
+
+import (
+	"github.com/diamondburned/cchat/text"
+	"github.com/yuin/goldmark/ast"
+)
+
+type CodeblockSegment struct {
+	start, end int
+	language   string
+}
+
+var _ text.Codeblocker = (*CodeblockSegment)(nil)
+
+func (r *TextRenderer) codeblock(n *ast.FencedCodeBlock, enter bool) ast.WalkStatus {
+	if enter {
+		// Create a segment.
+		seg := CodeblockSegment{
+			start:    r.i(),
+			language: string(n.Language(r.src)),
+		}
+
+		// Join all lines together.
+		var lines = n.Lines()
+
+		for i := 0; i < lines.Len(); i++ {
+			line := lines.At(i)
+			r.buf.Write(line.Value(r.src))
+		}
+
+		// Close the segment.
+		seg.end = r.i()
+		r.append(seg)
+	}
+
+	return ast.WalkContinue
+}
+
+func (c CodeblockSegment) Bounds() (start, end int) {
+	return c.start, c.end
+}
+
+func (c CodeblockSegment) CodeblockLanguage() string {
+	return c.language
+}

segments/emoji.go

@@ -0,0 +1,55 @@
+package segments
+
+import (
+	"github.com/diamondburned/cchat/text"
+	"github.com/diamondburned/ningen/md"
+	"github.com/yuin/goldmark/ast"
+)
+
+const (
+	InlineEmojiSize = 22
+	LargeEmojiSize  = 48
+)
+
+type EmojiSegment struct {
+	start    int
+	name     string
+	emojiURL string
+	large    bool
+}
+
+var _ text.Imager = (*EmojiSegment)(nil)
+
+func (r *TextRenderer) emoji(n *md.Emoji, enter bool) ast.WalkStatus {
+	if enter {
+		r.append(EmojiSegment{
+			start:    r.i(),
+			name:     n.Name,
+			large:    n.Large,
+			emojiURL: n.EmojiURL() + "&size=64",
+		})
+	}
+
+	return ast.WalkContinue
+}
+
+func (e EmojiSegment) Bounds() (start, end int) {
+	return e.start, e.start
+}
+
+func (e EmojiSegment) Image() string {
+	return e.emojiURL
+}
+
+// TODO: large emoji
+
+func (e EmojiSegment) ImageSize() (w, h int) {
+	if e.large {
+		return LargeEmojiSize, LargeEmojiSize
+	}
+	return InlineEmojiSize, InlineEmojiSize
+}
+
+func (e EmojiSegment) ImageText() string {
+	return ":" + e.name + ":"
+}

segments/inline_attr.go

@@ -0,0 +1,104 @@
+package segments
+
+import (
+	"github.com/diamondburned/cchat/text"
+	"github.com/diamondburned/ningen/md"
+	"github.com/yuin/goldmark/ast"
+)
+
+type inlineState struct {
+	// TODO: use a stack to allow overlapping
+	InlineSegment
+}
+
+func (i *inlineState) add(attr md.Attribute) {
+	if attr.Has(md.AttrBold) {
+		i.attributes |= text.AttrBold
+	}
+	if attr.Has(md.AttrItalics) {
+		i.attributes |= text.AttrItalics
+	}
+	if attr.Has(md.AttrUnderline) {
+		i.attributes |= text.AttrUnderline
+	}
+	if attr.Has(md.AttrStrikethrough) {
+		i.attributes |= text.AttrStrikethrough
+	}
+	if attr.Has(md.AttrSpoiler) {
+		i.attributes |= text.AttrSpoiler
+	}
+	if attr.Has(md.AttrMonospace) {
+		i.attributes |= text.AttrMonospace
+	}
+}
+
+func (i *inlineState) remove(attr md.Attribute) {
+	if attr.Has(md.AttrBold) {
+		i.attributes &= ^text.AttrBold
+	}
+	if attr.Has(md.AttrItalics) {
+		i.attributes &= ^text.AttrItalics
+	}
+	if attr.Has(md.AttrUnderline) {
+		i.attributes &= ^text.AttrUnderline
+	}
+	if attr.Has(md.AttrStrikethrough) {
+		i.attributes &= ^text.AttrStrikethrough
+	}
+	if attr.Has(md.AttrSpoiler) {
+		i.attributes &= ^text.AttrSpoiler
+	}
+	if attr.Has(md.AttrMonospace) {
+		i.attributes &= ^text.AttrMonospace
+	}
+}
+
+func (i inlineState) copy() InlineSegment {
+	return i.InlineSegment
+}
+
+type InlineSegment struct {
+	start, end int
+	attributes text.Attribute
+}
+
+var _ text.Attributor = (*InlineSegment)(nil)
+
+// inline parses an inline node. This method at the moment will always create a
+// new segment for overlapping attributes.
+func (r *TextRenderer) inline(n *md.Inline, enter bool) ast.WalkStatus {
+	// For instructions on how this works, refer to inline_attr.jpg.
+
+	// Pop the last segment if it's not empty.
+	if !r.inls.empty() {
+		r.inls.end = r.i()
+
+		// Only use this section if the length is not zero.
+		if r.inls.start != r.inls.end {
+			r.append(r.inls.copy())
+		}
+	}
+
+	if enter {
+		r.inls.add(n.Attr)
+	} else {
+		r.inls.remove(n.Attr)
+	}
+
+	// Update the start pointer of the current segment.
+	r.inls.start = r.i()
+
+	return ast.WalkContinue
+}
+
+func (i InlineSegment) Bounds() (start, end int) {
+	return i.start, i.end
+}
+
+func (i InlineSegment) Attribute() text.Attribute {
+	return i.attributes
+}
+
+func (i InlineSegment) empty() bool {
+	return i.attributes == 0 || i.start < i.end
+}

segments/link.go

@@ -0,0 +1,56 @@
+package segments
+
+import (
+	"github.com/diamondburned/cchat/text"
+	"github.com/yuin/goldmark/ast"
+)
+
+type LinkSegment struct {
+	start, end int
+	url        string
+}
+
+var _ text.Linker = (*LinkSegment)(nil)
+
+func (r *TextRenderer) link(n *ast.Link, enter bool) ast.WalkStatus {
+	if enter {
+		// Make a segment with a start pointing to the end of buffer.
+		seg := LinkSegment{
+			start: r.i(),
+			url:   string(n.Destination),
+		}
+
+		// Write the actual title.
+		r.buf.Write(n.Title)
+
+		// Close the segment.
+		seg.end = r.i()
+		r.append(seg)
+	}
+
+	return ast.WalkContinue
+}
+
+func (r *TextRenderer) autoLink(n *ast.AutoLink, enter bool) ast.WalkStatus {
+	if enter {
+		seg := LinkSegment{
+			start: r.i(),
+			url:   string(n.URL(r.src)),
+		}
+
+		r.buf.Write(n.URL(r.src))
+
+		seg.end = r.i()
+		r.append(seg)
+	}
+
+	return ast.WalkContinue
+}
+
+func (l LinkSegment) Bounds() (start, end int) {
+	return l.start, l.end
+}
+
+func (l LinkSegment) Link() (url string) {
+	return l.url
+}

segments/md.go

@@ -0,0 +1,101 @@
+package segments
+
+import (
+	"bytes"
+
+	"github.com/diamondburned/arikawa/discord"
+	"github.com/diamondburned/arikawa/state"
+	"github.com/diamondburned/cchat/text"
+	"github.com/diamondburned/ningen/md"
+	"github.com/yuin/goldmark/ast"
+)
+
+type TextRenderer struct {
+	buf  *bytes.Buffer
+	src  []byte
+	segs []text.Segment
+	inls inlineState
+}
+
+func ParseMessage(m *discord.Message, s state.Store) text.Rich {
+	return ParseWithMessage([]byte(m.Content), s, m, true)
+}
+
+func ParseWithMessage(b []byte, s state.Store, m *discord.Message, msg bool) text.Rich {
+	node := md.ParseWithMessage(b, s, m, msg)
+	return RenderNode(b, node)
+}
+
+func Parse(b []byte) text.Rich {
+	node := md.Parse(b)
+	return RenderNode(b, node)
+}
+
+func RenderNode(source []byte, n ast.Node) text.Rich {
+	buf := &bytes.Buffer{}
+	buf.Grow(len(source))
+
+	r := TextRenderer{
+		src:  source,
+		buf:  buf,
+		segs: make([]text.Segment, 0, n.ChildCount()),
+	}
+
+	ast.Walk(n, r.renderNode)
+
+	return text.Rich{
+		Content:  buf.String(),
+		Segments: r.segs,
+	}
+}
+
+// i returns the current cursor position.
+func (r *TextRenderer) i() int {
+	return r.buf.Len()
+}
+
+func (r *TextRenderer) append(segs ...text.Segment) {
+	r.segs = append(r.segs, segs...)
+}
+
+func (r *TextRenderer) renderNode(n ast.Node, enter bool) (ast.WalkStatus, error) {
+	switch n := n.(type) {
+	case *ast.Document:
+	case *ast.Paragraph:
+		if !enter {
+			// TODO: investigate
+			// r.buf.WriteByte('\n')
+		}
+	case *ast.Blockquote:
+		return r.blockquote(n, enter), nil
+	case *ast.FencedCodeBlock:
+		return r.codeblock(n, enter), nil
+	case *ast.Link:
+		return r.link(n, enter), nil
+	case *ast.AutoLink:
+		return r.autoLink(n, enter), nil
+	case *md.Inline:
+		return r.inline(n, enter), nil
+	case *md.Emoji:
+		return r.emoji(n, enter), nil
+	case *md.Mention:
+		return r.mention(n, enter), nil
+	case *ast.String:
+		if enter {
+			r.buf.Write(n.Value)
+		}
+	case *ast.Text:
+		if enter {
+			r.buf.Write(n.Segment.Value(r.src))
+
+			switch {
+			case n.HardLineBreak():
+				r.buf.WriteString("\n\n")
+			case n.SoftLineBreak():
+				r.buf.WriteByte('\n')
+			}
+		}
+	}
+
+	return ast.WalkContinue, nil
+}

segments/md_test.go

@@ -0,0 +1,123 @@
+package segments
+
+import (
+	"errors"
+	"log"
+	"testing"
+
+	"github.com/diamondburned/arikawa/discord"
+	"github.com/diamondburned/arikawa/state"
+	"github.com/diamondburned/cchat/text"
+	"github.com/go-test/deep"
+)
+
+type segtest struct {
+	in  string
+	out text.Rich
+}
+
+func mksegtest(in string, out string, segs ...text.Segment) segtest {
+	return segtest{
+		in:  in,
+		out: text.Rich{Content: out, Segments: segs},
+	}
+}
+
+func init() {
+	deep.CompareUnexportedFields = true
+}
+
+func TestParse(t *testing.T) {
+	var tests = []segtest{
+		mksegtest(
+			"This makes me <:Thonk:456835728559702052>",
+			"This makes me ",
+			EmojiSegment{
+				start:    14,
+				large:    false,
+				name:     "Thonk",
+				emojiURL: "https://cdn.discordapp.com/emojis/456835728559702052.png?v=1&size=64",
+			},
+		),
+		mksegtest(
+			"This is https://google.com",
+			"This is https://google.com",
+			LinkSegment{8, 26, "https://google.com"},
+		),
+		mksegtest(
+			"**bold and *italics*** text",
+			"bold and italics text",
+			InlineSegment{0, 9, text.AttrBold},
+			InlineSegment{9, 16, text.AttrBold | text.AttrItalics},
+		),
+		mksegtest(
+			"> imagine best trap\n> not being astolfo",
+			"> imagine best trap\n> not being astolfo",
+			BlockquoteSegment{0, 39},
+		),
+		mksegtest(
+			"```go\npackage main\n\nfunc main() {}```",
+			"package main\n\nfunc main() {}",
+			CodeblockSegment{0, 28, "go"},
+		),
+	}
+
+	for _, test := range tests {
+		text := Parse([]byte(test.in))
+		log.Printf("Output: %#v\n", text)
+
+		assert(t, text, test)
+	}
+}
+
+func TestMessage(t *testing.T) {
+	var msg = discord.Message{
+		ID:      69420,
+		Content: "<@1> where's <#2>",
+		Mentions: []discord.GuildUser{{
+			User: discord.User{
+				ID:       1,
+				Username: "astolfo",
+			},
+		}},
+	}
+
+	var store = mockStore{}
+
+	text := ParseMessage(&msg, store)
+	log.Printf("Output: %#v\n", text)
+
+	assert(t, text, mksegtest(
+		"Message",
+		"@astolfo where's #traps",
+		MentionSegment{0, 8},
+		MentionSegment{17, 23},
+	))
+}
+
+type mockStore struct {
+	state.NoopStore
+}
+
+func (mockStore) Channel(id discord.Snowflake) (*discord.Channel, error) {
+	if id != 2 {
+		return nil, errors.New("Unknown channel")
+	}
+
+	return &discord.Channel{
+		ID:   2,
+		Name: "traps",
+	}, nil
+}
+
+func assert(t *testing.T, got text.Rich, expect segtest) {
+	t.Helper()
+
+	if diff := deep.Equal(got, expect.out); diff != nil {
+		t.Logf("Got %d error(s) for %q", len(diff), expect.in)
+
+		for _, d := range diff {
+			t.Error("(got != expected) " + d)
+		}
+	}
+}

segments/mention.go

@@ -0,0 +1,48 @@
+package segments
+
+import (
+	"github.com/diamondburned/cchat/text"
+	"github.com/diamondburned/ningen/md"
+	"github.com/yuin/goldmark/ast"
+)
+
+const (
+	mentionChannel uint8 = iota
+	mentionUser
+	mentionRole
+)
+
+type MentionSegment struct {
+	start, end int
+}
+
+var _ text.Segment = (*MentionSegment)(nil)
+
+func (r *TextRenderer) mention(n *md.Mention, enter bool) ast.WalkStatus {
+	if enter {
+		seg := MentionSegment{start: r.i()}
+
+		switch {
+		case n.Channel != nil:
+			r.buf.WriteString("#" + n.Channel.Name)
+		case n.GuildUser != nil:
+			r.buf.WriteString("@" + n.GuildUser.Username)
+		case n.GuildRole != nil:
+			r.buf.WriteString("@" + n.GuildRole.Name)
+		}
+
+		seg.end = r.i()
+		r.append(seg)
+	}
+
+	return ast.WalkContinue
+}
+
+func (m MentionSegment) Bounds() (start, end int) {
+	return m.start, m.end
+}
+
+// TODO
+func (m MentionSegment) MentionInfo() text.Rich {
+	return text.Rich{}
+}