From c649a7675c3a59b9721f52706038d6772b751322 Mon Sep 17 00:00:00 2001 From: Matteo Kloiber Date: Sat, 4 Apr 2015 15:09:39 +0200 Subject: [PATCH] Added unicode support for markdown renderer. --- textRender.go | 62 +++++++++++++++++----- textRender_test.go | 128 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 151 insertions(+), 39 deletions(-) diff --git a/textRender.go b/textRender.go index 93b2e21..3cf0701 100644 --- a/textRender.go +++ b/textRender.go @@ -8,12 +8,13 @@ import ( // TextRender adds common methods for rendering a text on screeen. type TextRender interface { NormalizedText(text string) string + Render(lastColor, background Attribute) RenderedSequence RenderSequence(start, end int, lastColor, background Attribute) RenderedSequence } // MarkdownRegex is used by MarkdownTextRenderer to determine how to format the // text. -const MarkdownRegex = `(?:\[([[a-z]+)\])\(([a-z\s,]+)\)` +const MarkdownRegex = `(?:\[([^]]+)\])\(([a-z\s,]+)\)` // unexported because a pattern can't be a constant and we don't want anyone // messing with the regex. @@ -48,8 +49,13 @@ func (r MarkdownTextRenderer) normalizeText(text string) string { return text } +// Returns the position considering unicode characters. +func posUnicode(text string, pos int) int { + return len([]rune(text[:pos])) +} + /* -RenderSequence renders the sequence `text` using a markdown-like syntax: +Render renders the sequence `text` using a markdown-like syntax: `[hello](red) world` will become: `hello world` where hello is red. You may also specify other attributes such as bold text: @@ -60,10 +66,16 @@ For all available combinations, colors, and attribute, see: `StringToAttribute`. This method returns a RenderedSequence */ +func (r MarkdownTextRenderer) Render(lastColor, background Attribute) RenderedSequence { + return r.RenderSequence(0, -1, lastColor, background) +} + +// RenderSequence renders the text just like Render but the start and end can +// be specified. func (r MarkdownTextRenderer) RenderSequence(start, end int, lastColor, background Attribute) RenderedSequence { text := r.Text if end == -1 { - end = len(r.NormalizedText()) + end = len([]rune(r.NormalizedText())) } getMatch := func(s string) []int { @@ -81,6 +93,8 @@ func (r MarkdownTextRenderer) RenderSequence(start, end int, lastColor, backgrou color := StringToAttribute(text[colorStart:colorEnd]) content := text[contentStart:contentEnd] theSequence := ColorSubsequence{color, contentStart - 1, contentEnd - 1} + theSequence.Start = posUnicode(text, contentStart) - 1 + theSequence.End = posUnicode(text, contentEnd) - 1 if start < theSequence.End && end > theSequence.Start { // Make the sequence relative and append. @@ -105,7 +119,9 @@ func (r MarkdownTextRenderer) RenderSequence(start, end int, lastColor, backgrou if end == -1 { end = len(text) } - return RenderedSequence{text[start:end], lastColor, background, sequences} + + runes := []rune(text)[start:end] + return RenderedSequence{string(runes), lastColor, background, sequences, nil} } // RenderedSequence is a string sequence that is capable of returning the @@ -115,6 +131,9 @@ type RenderedSequence struct { LastColor Attribute BackgroundColor Attribute Sequences []ColorSubsequence + + // Use the color() method for getting the correct value. + mapSequences map[int]Attribute } // A ColorSubsequence represents a color for the given text span. @@ -137,22 +156,39 @@ func ColorSubsequencesToMap(sequences []ColorSubsequence) map[int]Attribute { return result } +func (s *RenderedSequence) colors() map[int]Attribute { + if s.mapSequences == nil { + s.mapSequences = ColorSubsequencesToMap(s.Sequences) + } + + return s.mapSequences +} + // Buffer returns the colorful formatted buffer and the last color that was // used. func (s *RenderedSequence) Buffer(x, y int) ([]Point, Attribute) { buffer := make([]Point, 0, len(s.NormalizedText)) // This is just an assumtion - colors := ColorSubsequencesToMap(s.Sequences) - for i, r := range []rune(s.NormalizedText) { - color, ok := colors[i] - if !ok { - color = s.LastColor - } - - p := Point{r, s.BackgroundColor, color, x, y} + for i := range []rune(s.NormalizedText) { + p, width := s.PointAt(i, x, y) buffer = append(buffer, p) - x += charWidth(r) + x += width } return buffer, s.LastColor } + +// PointAt returns the point at the position of n. The x, and y coordinates +// are used for placing the point at the right position. +// Since some charaters are wider (like `一`), this method also returns the +// width the point actually took. +// This is important for increasing the x property properly. +func (s *RenderedSequence) PointAt(n, x, y int) (Point, int) { + color, ok := s.colors()[n] + if !ok { + color = s.LastColor + } + + char := []rune(s.NormalizedText)[n] + return Point{char, s.BackgroundColor, color, x, y}, charWidth(char) +} diff --git a/textRender_test.go b/textRender_test.go index 219363b..388746f 100644 --- a/textRender_test.go +++ b/textRender_test.go @@ -6,6 +6,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMarkdownTextRenderer_normalizeText(t *testing.T) { @@ -20,9 +21,15 @@ func TestMarkdownTextRenderer_normalizeText(t *testing.T) { got = renderer.normalizeText("[foo](g) hello [bar]green (world)") assert.Equal(t, got, "foo hello [bar]green (world)") - // FIXME: [[ERROR]](red,bold) test should normalize to: - // [ERROR] test - // FIXME: Support unicode inside the error message. + got = "笀耔 [澉 灊灅甗](RED) 郔镺 笀耔 澉 [灊灅甗](yellow) 郔镺" + expected := "笀耔 澉 灊灅甗 郔镺 笀耔 澉 灊灅甗 郔镺" + assert.Equal(t, renderer.normalizeText(got), expected) + + got = renderer.normalizeText("[(foo)](red,white) bar") + assert.Equal(t, renderer.normalizeText(got), "(foo) bar") + + got = renderer.normalizeText("[[foo]](red,white) bar") + assert.Equal(t, renderer.normalizeText(got), "[foo] bar") } func TestMarkdownTextRenderer_NormalizedText(t *testing.T) { @@ -81,8 +88,28 @@ func TestMarkdownTextRenderer_RenderSequence(t *testing.T) { assertColorSubsequence(t, got.Sequences[1], "BLUE", 8, 9) } - // Test half-rendered text (unicode) - // FIXME: Add + // TODO: test barkets + + // Test with unicodes + text := "笀耔 [澉 灊灅甗](RED) 郔镺 笀耔 澉 [灊灅甗](yellow) 郔镺" + normalized := "笀耔 澉 灊灅甗 郔镺 笀耔 澉 灊灅甗 郔镺" + renderer = MarkdownTextRenderer{text} + got = renderer.RenderSequence(0, -1, 4, 7) + if assertRenderSequence(t, got, 4, 7, normalized, 2) { + assertColorSubsequence(t, got.Sequences[0], "RED", 3, 8) + assertColorSubsequence(t, got.Sequences[1], "YELLOW", 17, 20) + } + + got = renderer.RenderSequence(6, 7, 0, 0) + if assertRenderSequence(t, got, 0, 0, "灅", 1) { + assertColorSubsequence(t, got.Sequences[0], "RED", 0, 1) + } + + got = renderer.RenderSequence(7, 19, 0, 0) + if assertRenderSequence(t, got, 0, 0, "甗 郔镺 笀耔 澉 灊灅", 2) { + assertColorSubsequence(t, got.Sequences[0], "RED", 0, 1) + assertColorSubsequence(t, got.Sequences[1], "YELLOW", 10, 12) + } // Test inside renderer = MarkdownTextRenderer{"foo [foobar](red) bar"} @@ -92,6 +119,15 @@ func TestMarkdownTextRenderer_RenderSequence(t *testing.T) { } } +func TestMarkdownTextRenderer_Render(t *testing.T) { + renderer := MarkdownTextRenderer{"[foo](red,bold) [bar](blue)"} + got := renderer.Render(6, 8) + if assertRenderSequence(t, got, 6, 8, "foo bar", 2) { + assertColorSubsequence(t, got.Sequences[0], "RED,BOLD", 0, 3) + assertColorSubsequence(t, got.Sequences[1], "blue", 4, 7) + } +} + func TestColorSubsequencesToMap(t *testing.T) { colorSubsequences := []ColorSubsequence{ {ColorRed, 1, 4}, @@ -107,38 +143,78 @@ func TestColorSubsequencesToMap(t *testing.T) { assert.Equal(t, expected, ColorSubsequencesToMap(colorSubsequences)) } -func TestRenderedSequence_Buffer(t *testing.T) { +func getTestRenderedSequence() RenderedSequence { cs := []ColorSubsequence{ {ColorRed, 3, 5}, {ColorBlue | AttrBold, 9, 10}, } - sequence := RenderedSequence{"Hello world", ColorWhite, ColorBlack, cs} - newPoint := func(char string, x, y int, colorA ...Attribute) Point { - var color Attribute - if colorA != nil && len(colorA) == 1 { - color = colorA[0] - } else { - color = ColorWhite - } - return Point{[]rune(char)[0], ColorBlack, color, x, y} + return RenderedSequence{"Hello world", ColorWhite, ColorBlack, cs, nil} +} + +func newTestPoint(char rune, x, y int, colorA ...Attribute) Point { + var color Attribute + if colorA != nil && len(colorA) == 1 { + color = colorA[0] + } else { + color = ColorWhite } + return Point{char, ColorBlack, color, x, y} +} + +func TestRenderedSequence_Buffer(t *testing.T) { + sequence := getTestRenderedSequence() expected := []Point{ - newPoint("H", 5, 7), - newPoint("e", 6, 7), - newPoint("l", 7, 7), - newPoint("l", 7, 7, ColorRed), - newPoint("o", 8, 7, ColorRed), - newPoint(" ", 9, 7), - newPoint("w", 10, 7), - newPoint("o", 11, 7), - newPoint("r", 12, 7), - newPoint("l", 13, 7, ColorBlue|AttrBold), - newPoint("d", 14, 7), + newTestPoint('H', 5, 7), + newTestPoint('e', 6, 7), + newTestPoint('l', 7, 7), + newTestPoint('l', 7, 7, ColorRed), + newTestPoint('o', 8, 7, ColorRed), + newTestPoint(' ', 9, 7), + newTestPoint('w', 10, 7), + newTestPoint('o', 11, 7), + newTestPoint('r', 12, 7), + newTestPoint('l', 13, 7, ColorBlue|AttrBold), + newTestPoint('d', 14, 7), } + buffer, lastColor := sequence.Buffer(5, 7) assert.Equal(t, expected[:3], buffer[:3]) assert.Equal(t, ColorWhite, lastColor) } + +func AssertPoint(t *testing.T, got Point, char rune, x, y int, colorA ...Attribute) { + expected := newTestPoint(char, x, y, colorA...) + assert.Equal(t, expected, got) +} + +func TestRenderedSequence_PointAt(t *testing.T) { + sequence := getTestRenderedSequence() + pointAt := func(n, x, y int) Point { + p, w := sequence.PointAt(n, x, y) + assert.Equal(t, w, 1) + + return p + } + + AssertPoint(t, pointAt(0, 3, 4), 'H', 3, 4) + AssertPoint(t, pointAt(1, 2, 1), 'e', 2, 1) + AssertPoint(t, pointAt(2, 6, 3), 'l', 6, 3) + AssertPoint(t, pointAt(3, 8, 8), 'l', 8, 8, ColorRed) + AssertPoint(t, pointAt(4, 1, 4), 'o', 1, 4, ColorRed) + AssertPoint(t, pointAt(5, 3, 6), ' ', 3, 6) + AssertPoint(t, pointAt(6, 4, 3), 'w', 4, 3) + AssertPoint(t, pointAt(7, 5, 2), 'o', 5, 2) + AssertPoint(t, pointAt(8, 0, 5), 'r', 0, 5) + AssertPoint(t, pointAt(9, 9, 0), 'l', 9, 0, ColorBlue|AttrBold) + AssertPoint(t, pointAt(10, 7, 1), 'd', 7, 1) +} + +func TestPosUnicode(t *testing.T) { + // Every characters takes 3 bytes + text := "你好世界" + require.Equal(t, "你好", text[:6]) + assert.Equal(t, 2, posUnicode(text, 6)) +}