OrgGo/tcell
1
0
Fork 0
mirror of https://github.com/gdamore/tcell.git synced 2025-04-27 13:48:50 +08:00
Code Issues Projects Releases Wiki Activity

fixes #40 Use the new and better gdamore/encoding package

Browse Source
This commit is contained in:
Garrett D'Amore 2015-10-09 12:24:29 -07:00
parent 5e1c42d96d
commit 10c966885c
7 changed files with 9 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
ascii.go
View File

@ -1,91 +0,0 @@
// Copyright 2015 The TCell Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcell
import (
"unicode/utf8"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
)
type ascii struct{ transform.NopResetter }
type asciiDecoder struct{ transform.NopResetter }
type asciiEncoder struct{ transform.NopResetter }
// ASCII represents an basic 7-bit ASCII scheme. It decodes directly to UTF-8
// without change, as all ASCII values are legal UTF-8. It encodes any UTF-8
// runes outside of ASCII to 0x1A, the ASCII substitution character.
var ASCII encoding.Encoding = ascii{}
func (ascii) NewDecoder() transform.Transformer {
return asciiDecoder{}
}
func (ascii) NewEncoder() transform.Transformer {
return asciiEncoder{}
}
func (asciiDecoder) Transform(dst, src []byte, atEOF bool) (int, int, error) {
var e error
var ndst, nsrc int
for _, c := range src {
if ndst >= len(dst) {
e = transform.ErrShortDst
break
}
dst[ndst] = c
ndst++
nsrc++
}
return ndst, nsrc, e
}
func (asciiEncoder) Transform(dst, src []byte, atEOF bool) (int, int, error) {
var e error
var ndst, nsrc int
var sz int
for nsrc < len(src) {
if ndst >= len(dst) {
e = transform.ErrShortDst
break
}
r := rune(src[nsrc])
if r < utf8.RuneSelf {
dst[ndst] = uint8(r)
nsrc++
ndst++
continue
}
// No valid runes beyond ASCII. However, we need to consume
// the full rune, and report incomplete runes properly.
// Attempt to decode a multibyte rune
r, sz = utf8.DecodeRune(src[nsrc:])
if sz == 1 {
// If its inconclusive due to insufficient data in
// in the source, report it
if !atEOF && !utf8.FullRune(src[nsrc:]) {
e = transform.ErrShortSrc
break
}
}
nsrc += sz
dst[ndst] = encoding.ASCIISub
ndst++
}
return ndst, nsrc, e
}

8
encoding.go
View File

@ -19,6 +19,8 @@ import (
"sync" "sync"
"golang.org/x/text/encoding" "golang.org/x/text/encoding"
gencoding "github.com/gdamore/encoding"
) )
var encodings map[string]encoding.Encoding var encodings map[string]encoding.Encoding
@ -119,7 +121,7 @@ func GetEncoding(charset string) encoding.Encoding {
} }
switch encodingFallback { switch encodingFallback {
case EncodingFallbackASCII: case EncodingFallbackASCII:
return ASCII return gencoding.ASCII
case EncodingFallbackUTF8: case EncodingFallbackUTF8:
return encoding.Nop return encoding.Nop
} }
@ -129,6 +131,6 @@ func GetEncoding(charset string) encoding.Encoding {
func init() { func init() {
// We always support UTF-8 and ASCII. // We always support UTF-8 and ASCII.
encodings = make(map[string]encoding.Encoding) encodings = make(map[string]encoding.Encoding)
encodings["utf-8"] = UTF8 encodings["utf-8"] = gencoding.UTF8
encodings["us-ascii"] = ASCII encodings["us-ascii"] = gencoding.ASCII
} }

5
encoding/all.go
View File

@ -15,6 +15,7 @@
package encoding package encoding
import ( import (
"github.com/gdamore/encoding"
"github.com/gdamore/tcell" "github.com/gdamore/tcell"
"golang.org/x/text/encoding/charmap" "golang.org/x/text/encoding/charmap"
@ -30,8 +31,8 @@ import (
// are rather large (particularly those from East Asia.) // are rather large (particularly those from East Asia.)
func Register() { func Register() {
// We supply latin1 and latin5, because Go doesn't // We supply latin1 and latin5, because Go doesn't
tcell.RegisterEncoding("ISO8859-1", ISO8859_1) tcell.RegisterEncoding("ISO8859-1", encoding.ISO8859_1)
tcell.RegisterEncoding("ISO8859-9", ISO8859_9) tcell.RegisterEncoding("ISO8859-9", encoding.ISO8859_9)
tcell.RegisterEncoding("ISO8859-10", charmap.ISO8859_10) tcell.RegisterEncoding("ISO8859-10", charmap.ISO8859_10)
tcell.RegisterEncoding("ISO8859-13", charmap.ISO8859_13) tcell.RegisterEncoding("ISO8859-13", charmap.ISO8859_13)

144
encoding/charmap.go
View File

@ -1,144 +0,0 @@
// Copyright 2015 The TCell Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package encoding
import (
"unicode/utf8"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
)
// suitable for 8-bit encodings
type cmap struct {
transform.NopResetter
bytes map[rune]byte
runes [256]rune // offset by 128, as all values are identical
ascii bool
}
type cmapDecoder struct {
transform.NopResetter
cmap *cmap
}
type cmapEncoder struct {
transform.NopResetter
cmap *cmap
}
func (c *cmap) Init() {
c.bytes = make(map[rune]byte)
for i := 0; i < 256; i++ {
c.bytes[rune(i)] = byte(i)
c.runes[i] = rune(i)
c.ascii = true
}
}
func (c *cmap) Map(b byte, r rune) {
if b < 128 {
c.ascii = false
}
// delete the old self-mapping
delete(c.bytes, rune(b))
// and add the new one
c.bytes[r] = b
c.runes[int(b)] = r
}
func (c *cmap) NewDecoder() transform.Transformer {
return cmapDecoder{cmap: c}
}
func (c *cmap) NewEncoder() transform.Transformer {
return cmapEncoder{cmap: c}
}
func (d cmapDecoder) Transform(dst, src []byte, atEOF bool) (int, int, error) {
var e error
var ndst, nsrc int
for _, c := range src {
if d.cmap.ascii && c < utf8.RuneSelf {
if ndst >= len(dst) {
e = transform.ErrShortDst
break
}
dst[ndst] = c
ndst++
nsrc++
continue
}
r := d.cmap.runes[c]
l := utf8.RuneLen(r)
// l will be a positive number, because we never inject invalid
// runes into the rune map.
if ndst+l > len(dst) {
e = transform.ErrShortDst
break
}
utf8.EncodeRune(dst[ndst:], r)
ndst += l
nsrc++
}
return ndst, nsrc, e
}
func (d cmapEncoder) Transform(dst, src []byte, atEOF bool) (int, int, error) {
var e error
var ndst, nsrc int
for nsrc < len(src) {
if ndst >= len(dst) {
e = transform.ErrShortDst
break
}
ch := src[nsrc]
if d.cmap.ascii && ch < utf8.RuneSelf {
dst[ndst] = ch
nsrc++
ndst++
continue
}
// No valid runes beyond 0xFF. However, we need to consume
// the full rune, and report incomplete runes properly.
// Attempt to decode a multibyte rune
r, sz := utf8.DecodeRune(src[nsrc:])
if r == utf8.RuneError && sz == 1 {
// If its inconclusive due to insufficient data in
// in the source, report it
if !atEOF && !utf8.FullRune(src[nsrc:]) {
e = transform.ErrShortSrc
break
}
}
if c, ok := d.cmap.bytes[r]; ok {
dst[ndst] = c
} else {
dst[ndst] = encoding.ASCIISub
}
nsrc += sz
ndst++
}
return ndst, nsrc, e
}

34
encoding/latin1.go
View File

@ -1,34 +0,0 @@
// Copyright 2015 The TCell Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package encoding
import (
"golang.org/x/text/encoding"
)
// ISO8859_1 represents the 8-bit ISO8859-1 scheme. It decodes directly to
// UTF-8 without change, as all ISO8859-1 values are legal UTF-8.
// Unicode values less than 256 (i.e. 8 bits) map 1:1 with 8859-1.
// It encodes runes outside of that to 0x1A, the ASCII substitution character.
var ISO8859_1 encoding.Encoding
func init() {
cm := &cmap{}
cm.Init()
// No further mapping needed for ISO8859-1, as there is exactly a 1:1
// mapping between the Unicode and 8859-1 namespaces.
ISO8859_1 = cm
}

35
encoding/latin5.go
View File

@ -1,35 +0,0 @@
// Copyright 2015 The TCell Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package encoding
import (
"golang.org/x/text/encoding"
)
// ISO8859_9 represents the 8-bit ISO8859-1 scheme. It decodes to UTF-8
// unchanged for all 256 positions except for six positions.
var ISO8859_9 encoding.Encoding
func init() {
cm := &cmap{}
cm.Init()
cm.Map(0xD0, 'Ğ')
cm.Map(0xDD, 'İ')
cm.Map(0xDE, 'Ş')
cm.Map(0xF0, 'ğ')
cm.Map(0xFD, 'ı')
cm.Map(0xFE, 'ş')
ISO8859_9 = cm
}

34
utf8.go
View File

@ -1,34 +0,0 @@
// Copyright 2015 The TCell Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcell
import (
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
)
type validUtf8 struct{}
// UTF8 is an encoding for UTF-8. All it does is verify that the UTF-8
// in is valid.
var UTF8 encoding.Encoding = validUtf8{}
func (validUtf8) NewDecoder() transform.Transformer {
return encoding.UTF8Validator
}
func (validUtf8) NewEncoder() transform.Transformer {
return encoding.UTF8Validator
}