Refactored encoding and stream support with interfaces. Added encoding support and encoding content streams for PDF generation.

pdf/core/encoding.go

@@ -0,0 +1,676 @@
+/*
+ * This file is subject to the terms and conditions defined in
+ * file 'LICENSE.md', which is part of this source code package.
+ */
+
+package core
+
+// Implement encoders for PDF. Currently supported:
+// - Raw (Identity)
+// - FlateDecode
+// - LZW
+// - ASCII Hex
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/hex"
+	"fmt"
+	"io"
+
+	// Need two slightly different implementations of LZW (EarlyChange parameter).
+	lzw0 "compress/lzw"
+	lzw1 "golang.org/x/image/tiff/lzw"
+
+	"github.com/unidoc/unidoc/common"
+)
+
+type StreamEncoder interface {
+	EncodeBytes(data []byte) ([]byte, error)
+	DecodeBytes(encoded []byte) ([]byte, error)
+	DecodeStream(streamObj *PdfObjectStream) ([]byte, error)
+	MakeEncodingDict() *PdfObjectDictionary
+}
+
+// Flate encoding.
+type FlateEncoder struct {
+	Predictor        int
+	BitsPerComponent int
+	// For predictors
+	Columns int
+	Colors  int
+}
+
+// Make a new flate encoder with default parameters, predictor 1 and bits per component 8.
+func NewFlateEncoder() *FlateEncoder {
+	encoder := &FlateEncoder{}
+
+	// Default (No prediction)
+	encoder.Predictor = 1
+
+	// Currently only supporting 8.
+	encoder.BitsPerComponent = 8
+
+	encoder.Colors = 1
+	encoder.Columns = 1
+
+	return encoder
+}
+
+// Make a new instance of an encoding dictionary for a stream object.
+// Has the Filter set and the DecodeParms.
+func (this *FlateEncoder) MakeEncodingDict() *PdfObjectDictionary {
+	dict := PdfObjectDictionary{}
+
+	dict["Filter"] = MakeName("FlateDecode")
+
+	if this.Predictor > 1 {
+		decodeParams := PdfObjectDictionary{}
+		decodeParams["Predictor"] = MakeInteger(int64(this.Predictor))
+		decodeParams["BitsPerComponent"] = MakeInteger(int64(this.BitsPerComponent))
+		decodeParams["Columns"] = MakeInteger(int64(this.Columns))
+		decodeParams["Colors"] = MakeInteger(int64(this.BitsPerComponent))
+		dict["DecodeParms"] = &decodeParams
+	}
+
+	return &dict
+}
+
+// Create a new flate decoder from a stream object, getting all the encoding parameters
+// from the DecodeParms stream object dictionary entry.
+func newFlateEncoderFromStream(streamObj *PdfObjectStream) (*FlateEncoder, error) {
+	encoder := NewFlateEncoder()
+
+	encDict := streamObj.PdfObjectDictionary
+	if encDict == nil {
+		// No encoding dictionary.
+		return encoder, nil
+	}
+
+	obj := (*encDict)["DecodeParms"]
+	if obj == nil {
+		// No decode parameters (optional).
+		return encoder, nil
+	}
+
+	decodeParams, isDict := obj.(*PdfObjectDictionary)
+	if !isDict {
+		common.Log.Debug("Error: DecodeParms not a dictionary (%T)", obj)
+		// No decode parameters.
+		return nil, fmt.Errorf("Invalid DecodeParms")
+	}
+
+	common.Log.Debug("decode params: %s", decodeParams.String())
+	obj, has := (*decodeParams)["Predictor"]
+	if !has {
+		common.Log.Debug("Error: Predictor missing from DecodeParms - Continue with default (1)")
+	} else {
+		predictor, ok := obj.(*PdfObjectInteger)
+		if !ok {
+			common.Log.Debug("Error: Predictor specified but not numeric (%T)", obj)
+			return nil, fmt.Errorf("Invalid Predictor")
+		}
+		encoder.Predictor = int(*predictor)
+	}
+
+	// Bits per component.  Use default if not specified (8).
+	obj, has = (*decodeParams)["BitsPerComponent"]
+	if has {
+		bpc, ok := obj.(*PdfObjectInteger)
+		if !ok {
+			common.Log.Debug("ERROR: Invalid BitsPerComponent")
+			return nil, fmt.Errorf("Invalid BitsPerComponent")
+		}
+		encoder.BitsPerComponent = int(*bpc)
+	}
+
+	if encoder.Predictor > 1 {
+		// Columns.
+		encoder.Columns = 1
+		obj, has = (*decodeParams)["Columns"]
+		if has {
+			columns, ok := obj.(*PdfObjectInteger)
+			if !ok {
+				return nil, fmt.Errorf("Predictor column invalid")
+			}
+
+			encoder.Columns = int(*columns)
+		}
+
+		// Colors.
+		// Number of interleaved color components per sample (Default 1 if not specified)
+		encoder.Colors = 1
+		obj, has = (*decodeParams)["Colors"]
+		if has {
+			colors, ok := obj.(*PdfObjectInteger)
+			if !ok {
+				return nil, fmt.Errorf("Predictor colors not an integer")
+			}
+			encoder.Colors = int(*colors)
+		}
+	}
+
+	return encoder, nil
+}
+
+func (this *FlateEncoder) DecodeBytes(encoded []byte) ([]byte, error) {
+	bufReader := bytes.NewReader(encoded)
+	r, err := zlib.NewReader(bufReader)
+	if err != nil {
+		common.Log.Debug("Decoding error %v\n", err)
+		common.Log.Debug("Stream (%d) % x", len(encoded), encoded)
+		return nil, err
+	}
+	defer r.Close()
+
+	var outBuf bytes.Buffer
+	outBuf.ReadFrom(r)
+	return outBuf.Bytes(), nil
+}
+
+// Decode a FlateEncoded stream object and give back decoded bytes.
+func (this *FlateEncoder) DecodeStream(streamObj *PdfObjectStream) ([]byte, error) {
+	// TODO: Revamp this support to handle TIFF predictor (2).
+	// Also handle more filter bytes and support more values of BitsPerComponent.
+
+	common.Log.Debug("FlateDecode")
+	common.Log.Debug("Predictor: %d", this.Predictor)
+	if this.BitsPerComponent != 8 {
+		return nil, fmt.Errorf("Invalid BitsPerComponent (only 8 supported)")
+	}
+
+	outData, err := this.DecodeBytes(streamObj.Stream)
+	if err != nil {
+		return nil, err
+	}
+	common.Log.Debug("En: % x\n", streamObj.Stream)
+	common.Log.Debug("De: % x\n", outData)
+
+	if this.Predictor > 1 {
+		if this.Predictor == 2 { // TIFF encoding: Needs some tests.
+			common.Log.Debug("Tiff encoding")
+			common.Log.Debug("Colors: %d", this.Colors)
+
+			rowLength := int(this.Columns) * this.Colors
+			rows := len(outData) / rowLength
+			if len(outData)%rowLength != 0 {
+				common.Log.Debug("ERROR: TIFF encoding: Invalid row length...")
+				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
+			}
+
+			if rowLength%this.Colors != 0 {
+				return nil, fmt.Errorf("Invalid row length (%d) for colors %d", rowLength, this.Colors)
+			}
+			common.Log.Debug("inp outData (%d): % x", len(outData), outData)
+
+			pOutBuffer := bytes.NewBuffer(nil)
+
+			// 0-255  -255 255 ; 0-255=-255;
+			for i := 0; i < rows; i++ {
+				rowData := outData[rowLength*i : rowLength*(i+1)]
+				//common.Log.Debug("RowData before: % d", rowData)
+				// Predicts the same as the sample to the left.
+				// Interleaved by colors.
+				for j := this.Colors; j < rowLength; j++ {
+					rowData[j] = byte(int(rowData[j]+rowData[j-this.Colors]) % 256)
+				}
+				pOutBuffer.Write(rowData)
+			}
+			pOutData := pOutBuffer.Bytes()
+			common.Log.Debug("POutData (%d): % x", len(pOutData), pOutData)
+			return pOutData, nil
+		} else if this.Predictor >= 10 && this.Predictor <= 15 {
+			common.Log.Debug("PNG Encoding")
+			rowLength := int(this.Columns + 1) // 1 byte to specify predictor algorithms per row.
+			rows := len(outData) / rowLength
+			if len(outData)%rowLength != 0 {
+				common.Log.Debug("ERROR: Invalid row length...")
+				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
+			}
+
+			pOutBuffer := bytes.NewBuffer(nil)
+
+			common.Log.Debug("Predictor columns: %d", this.Columns)
+			common.Log.Debug("Length: %d / %d = %d rows", len(outData), rowLength, rows)
+			prevRowData := make([]byte, rowLength)
+			for i := 0; i < rowLength; i++ {
+				prevRowData[i] = 0
+			}
+
+			for i := 0; i < rows; i++ {
+				rowData := outData[rowLength*i : rowLength*(i+1)]
+
+				fb := rowData[0]
+				switch fb {
+				case 0:
+					// No prediction. (No operation).
+				case 1:
+					// Sub: Predicts the same as the sample to the left.
+					for j := 2; j < rowLength; j++ {
+						rowData[j] = byte(int(rowData[j]+rowData[j-1]) % 256)
+					}
+				case 2:
+					// Up: Predicts the same as the sample above
+					for j := 1; j < rowLength; j++ {
+						rowData[j] = byte(int(rowData[j]+prevRowData[j]) % 256)
+					}
+				default:
+					common.Log.Debug("ERROR: Invalid filter byte (%d)", fb)
+					return nil, fmt.Errorf("Invalid filter byte (%d)", fb)
+				}
+
+				for i := 0; i < rowLength; i++ {
+					prevRowData[i] = rowData[i]
+				}
+				pOutBuffer.Write(rowData[1:])
+			}
+			pOutData := pOutBuffer.Bytes()
+			return pOutData, nil
+		} else {
+			common.Log.Debug("ERROR: Unsupported predictor (%d)", this.Predictor)
+			return nil, fmt.Errorf("Unsupported predictor (%d)", this.Predictor)
+		}
+	}
+
+	return outData, nil
+}
+
+// Encode a bytes array and return the encoded value based on the encoder parameters.
+func (this *FlateEncoder) EncodeBytes(data []byte) ([]byte, error) {
+	if this.Predictor != 1 {
+		return nil, fmt.Errorf("FlateEncoder Predictor = 1 only supported yet")
+	}
+
+	var b bytes.Buffer
+	w := zlib.NewWriter(&b)
+	w.Write(data)
+	w.Close()
+
+	return b.Bytes(), nil
+}
+
+// LZW encoding/decoding functionality.
+type LZWEncoder struct {
+	Predictor        int
+	BitsPerComponent int
+	// For predictors
+	Columns int
+	Colors  int
+	// LZW algorithm setting.
+	EarlyChange int
+}
+
+// Make a new LZW encoder with default parameters.
+func NewLZWEncoder() *LZWEncoder {
+	encoder := &LZWEncoder{}
+
+	// Default (No prediction)
+	encoder.Predictor = 1
+
+	// Currently only supporting 8.
+	encoder.BitsPerComponent = 8
+
+	encoder.Colors = 1
+	encoder.Columns = 1
+	encoder.EarlyChange = 1
+
+	return encoder
+}
+
+// Make a new instance of an encoding dictionary for a stream object.
+// Has the Filter set and the DecodeParms.
+func (this *LZWEncoder) MakeEncodingDict() *PdfObjectDictionary {
+	dict := PdfObjectDictionary{}
+
+	dict["Filter"] = MakeName("LZWDecode")
+
+	if this.Predictor > 1 {
+		decodeParams := PdfObjectDictionary{}
+		decodeParams["Predictor"] = MakeInteger(int64(this.Predictor))
+		decodeParams["BitsPerComponent"] = MakeInteger(int64(this.BitsPerComponent))
+		decodeParams["Columns"] = MakeInteger(int64(this.Columns))
+		decodeParams["Colors"] = MakeInteger(int64(this.BitsPerComponent))
+		dict["DecodeParms"] = &decodeParams
+	}
+
+	dict["EarlyChange"] = MakeInteger(int64(this.EarlyChange))
+
+	return &dict
+}
+
+// Create a new LZW encoder/decoder from a stream object, getting all the encoding parameters
+// from the DecodeParms stream object dictionary entry.
+func newLZWEncoderFromStream(streamObj *PdfObjectStream) (*LZWEncoder, error) {
+	// Start with default settings.
+	encoder := NewLZWEncoder()
+
+	encDict := streamObj.PdfObjectDictionary
+	if encDict == nil {
+		// No encoding dictionary.
+		return encoder, nil
+	}
+
+	obj := (*encDict)["DecodeParms"]
+	if obj == nil {
+		// No decode parameters (optional).
+		return encoder, nil
+	}
+
+	decodeParams, isDict := obj.(*PdfObjectDictionary)
+	if !isDict {
+		common.Log.Debug("Error: DecodeParms not a dictionary (%T)", obj)
+		return nil, fmt.Errorf("Invalid DecodeParms")
+	}
+
+	// The EarlyChange indicates when to increase code length, as different
+	// implementations use a different mechanisms. Essentially this chooses
+	// which LZW implementation to use.
+	// The default is 1 (one code early)
+	obj, has := (*decodeParams)["EarlyChange"]
+	if has {
+		earlyChange, ok := obj.(*PdfObjectInteger)
+		if !ok {
+			common.Log.Debug("Error: EarlyChange specified but not numeric (%T)", obj)
+			return nil, fmt.Errorf("Invalid EarlyChange")
+		}
+		if *earlyChange != 0 && *earlyChange != 1 {
+			return nil, fmt.Errorf("Invalid EarlyChange value (not 0 or 1)")
+		}
+
+		encoder.EarlyChange = int(*earlyChange)
+	}
+
+	obj, has = (*decodeParams)["Predictor"]
+	if has {
+		predictor, ok := obj.(*PdfObjectInteger)
+		if !ok {
+			common.Log.Debug("Error: Predictor specified but not numeric (%T)", obj)
+			return nil, fmt.Errorf("Invalid Predictor")
+		}
+		encoder.Predictor = int(*predictor)
+	}
+
+	// Bits per component.  Use default if not specified (8).
+	obj, has = (*decodeParams)["BitsPerComponent"]
+	if has {
+		bpc, ok := obj.(*PdfObjectInteger)
+		if !ok {
+			common.Log.Debug("ERROR: Invalid BitsPerComponent")
+			return nil, fmt.Errorf("Invalid BitsPerComponent")
+		}
+		encoder.BitsPerComponent = int(*bpc)
+	}
+
+	if encoder.Predictor > 1 {
+		// Columns.
+		encoder.Columns = 1
+		obj, has = (*decodeParams)["Columns"]
+		if has {
+			columns, ok := obj.(*PdfObjectInteger)
+			if !ok {
+				return nil, fmt.Errorf("Predictor column invalid")
+			}
+
+			encoder.Columns = int(*columns)
+		}
+
+		// Colors.
+		// Number of interleaved color components per sample (Default 1 if not specified)
+		encoder.Colors = 1
+		obj, has = (*decodeParams)["Colors"]
+		if has {
+			colors, ok := obj.(*PdfObjectInteger)
+			if !ok {
+				return nil, fmt.Errorf("Predictor colors not an integer")
+			}
+			encoder.Colors = int(*colors)
+		}
+	}
+
+	common.Log.Debug("decode params: %s", decodeParams.String())
+	return encoder, nil
+}
+
+func (this *LZWEncoder) DecodeBytes(encoded []byte) ([]byte, error) {
+	var outBuf bytes.Buffer
+	bufReader := bytes.NewReader(encoded)
+
+	var r io.ReadCloser
+	if this.EarlyChange == 1 {
+		// LZW implementation with code length increases one code early (1).
+		r = lzw1.NewReader(bufReader, lzw1.MSB, 8)
+	} else {
+		// 0: LZW implementation with postponed code length increases (0).
+		r = lzw0.NewReader(bufReader, lzw0.MSB, 8)
+	}
+	defer r.Close()
+
+	_, err := outBuf.ReadFrom(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return outBuf.Bytes(), nil
+}
+
+func (this *LZWEncoder) DecodeStream(streamObj *PdfObjectStream) ([]byte, error) {
+	// Revamp this support to handle TIFF predictor (2).
+	// Also handle more filter bytes and check
+	// BitsPerComponent.  Default value is 8, currently we are only
+	// supporting that one.
+
+	common.Log.Debug("LZW Decoding")
+	common.Log.Debug("Predictor: %d", this.Predictor)
+
+	outData, err := this.DecodeBytes(streamObj.Stream)
+	if err != nil {
+		return nil, err
+	}
+
+	common.Log.Debug(" IN: (%d) % x", len(streamObj.Stream), streamObj.Stream)
+	common.Log.Debug("OUT: (%d) % x", len(outData), outData)
+
+	if this.Predictor > 1 {
+		if this.Predictor == 2 { // TIFF encoding: Needs some tests.
+			common.Log.Debug("Tiff encoding")
+
+			rowLength := int(this.Columns) * this.Colors
+			rows := len(outData) / rowLength
+			if len(outData)%rowLength != 0 {
+				common.Log.Debug("ERROR: TIFF encoding: Invalid row length...")
+				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
+			}
+
+			if rowLength%this.Colors != 0 {
+				return nil, fmt.Errorf("Invalid row length (%d) for colors %d", rowLength, this.Colors)
+			}
+			common.Log.Debug("inp outData (%d): % x", len(outData), outData)
+
+			pOutBuffer := bytes.NewBuffer(nil)
+
+			// 0-255  -255 255 ; 0-255=-255;
+			for i := 0; i < rows; i++ {
+				rowData := outData[rowLength*i : rowLength*(i+1)]
+				//common.Log.Debug("RowData before: % d", rowData)
+				// Predicts the same as the sample to the left.
+				// Interleaved by colors.
+				for j := this.Colors; j < rowLength; j++ {
+					rowData[j] = byte(int(rowData[j]+rowData[j-this.Colors]) % 256)
+				}
+				// GH: Appears that this is not working as expected...
+
+				pOutBuffer.Write(rowData)
+			}
+			pOutData := pOutBuffer.Bytes()
+			common.Log.Debug("POutData (%d): % x", len(pOutData), pOutData)
+			return pOutData, nil
+		} else if this.Predictor >= 10 && this.Predictor <= 15 {
+			common.Log.Debug("PNG Encoding")
+			rowLength := int(this.Columns + 1) // 1 byte to specify predictor algorithms per row.
+			rows := len(outData) / rowLength
+			if len(outData)%rowLength != 0 {
+				common.Log.Debug("ERROR: Invalid row length...")
+				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
+			}
+
+			pOutBuffer := bytes.NewBuffer(nil)
+
+			common.Log.Debug("Predictor columns: %d", this.Columns)
+			common.Log.Debug("Length: %d / %d = %d rows", len(outData), rowLength, rows)
+			prevRowData := make([]byte, rowLength)
+			for i := 0; i < rowLength; i++ {
+				prevRowData[i] = 0
+			}
+
+			for i := 0; i < rows; i++ {
+				rowData := outData[rowLength*i : rowLength*(i+1)]
+
+				fb := rowData[0]
+				switch fb {
+				case 0:
+					// No prediction. (No operation).
+				case 1:
+					// Sub: Predicts the same as the sample to the left.
+					for j := 2; j < rowLength; j++ {
+						rowData[j] = byte(int(rowData[j]+rowData[j-1]) % 256)
+					}
+				case 2:
+					// Up: Predicts the same as the sample above
+					for j := 1; j < rowLength; j++ {
+						rowData[j] = byte(int(rowData[j]+prevRowData[j]) % 256)
+					}
+				default:
+					common.Log.Debug("ERROR: Invalid filter byte (%d)", fb)
+					return nil, fmt.Errorf("Invalid filter byte (%d)", fb)
+				}
+
+				for i := 0; i < rowLength; i++ {
+					prevRowData[i] = rowData[i]
+				}
+				pOutBuffer.Write(rowData[1:])
+			}
+			pOutData := pOutBuffer.Bytes()
+			return pOutData, nil
+		} else {
+			common.Log.Debug("ERROR: Unsupported predictor (%d)", this.Predictor)
+			return nil, fmt.Errorf("Unsupported predictor (%d)", this.Predictor)
+		}
+	}
+
+	return outData, nil
+}
+
+// Support for encoding LZW.  Currently not supporting predictors (raw compressed data only).
+// Only supports the Early change = 1 algorithm (compress/lzw) as the other implementation
+// does not have a write method.
+// TODO: Consider refactoring compress/lzw to allow both.
+func (this *LZWEncoder) EncodeBytes(data []byte) ([]byte, error) {
+	if this.Predictor != 1 {
+		return nil, fmt.Errorf("LZW Predictor = 1 only supported yet")
+	}
+
+	if this.EarlyChange == 1 {
+		return nil, fmt.Errorf("LZW Early Change = 0 only supported yet")
+	}
+
+	var b bytes.Buffer
+	w := lzw0.NewWriter(&b, lzw0.MSB, 8)
+	w.Write(data)
+	w.Close()
+
+	return b.Bytes(), nil
+}
+
+/////
+// ASCII hex encoder/decoder.
+type ASCIIHexEncoder struct {
+}
+
+// Make a new LZW encoder with default parameters.
+func NewASCIIHexEncoder() *ASCIIHexEncoder {
+	encoder := &ASCIIHexEncoder{}
+	return encoder
+}
+
+// Make a new instance of an encoding dictionary for a stream object.
+// Has the Filter set and the DecodeParms.
+func (this *ASCIIHexEncoder) MakeEncodingDict() *PdfObjectDictionary {
+	dict := PdfObjectDictionary{}
+
+	dict["Filter"] = MakeName("ASCIIHexDecode")
+	return &dict
+}
+
+func (this *ASCIIHexEncoder) DecodeBytes(encoded []byte) ([]byte, error) {
+	bufReader := bytes.NewReader(encoded)
+	inb := []byte{}
+	for {
+		b, err := bufReader.ReadByte()
+		if err != nil {
+			return nil, err
+		}
+		if b == '>' {
+			break
+		}
+		if IsWhiteSpace(b) {
+			continue
+		}
+		if (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F') || (b >= '0' && b <= '9') {
+			inb = append(inb, b)
+		} else {
+			common.Log.Debug("ERROR: Invalid ascii hex character (%c)", b)
+			return nil, fmt.Errorf("Invalid ascii hex character (%c)", b)
+		}
+	}
+	if len(inb)%2 == 1 {
+		inb = append(inb, '0')
+	}
+	common.Log.Debug("Inbound %s", inb)
+	outb := make([]byte, hex.DecodedLen(len(inb)))
+	_, err := hex.Decode(outb, inb)
+	if err != nil {
+		return nil, err
+	}
+	return outb, nil
+}
+
+// ASCII hex decoding.
+func (this *ASCIIHexEncoder) DecodeStream(streamObj *PdfObjectStream) ([]byte, error) {
+	return this.DecodeBytes(streamObj.Stream)
+}
+
+func (this *ASCIIHexEncoder) EncodeBytes(data []byte) ([]byte, error) {
+	var encoded bytes.Buffer
+
+	for _, b := range data {
+		encoded.WriteString(fmt.Sprintf("%.2X ", b))
+	}
+	encoded.WriteByte('>')
+
+	return encoded.Bytes(), nil
+}
+
+//
+// Raw encoder/decoder (no encoding, pass through)
+//
+type RawEncoder struct{}
+
+func NewRawEncoder() *RawEncoder {
+	return &RawEncoder{}
+}
+
+func (this *RawEncoder) MakeEncodingDict() *PdfObjectDictionary {
+	return &PdfObjectDictionary{}
+}
+
+func (this *RawEncoder) DecodeBytes(encoded []byte) ([]byte, error) {
+	return encoded, nil
+}
+
+func (this *RawEncoder) DecodeStream(streamObj *PdfObjectStream) ([]byte, error) {
+	return streamObj.Stream, nil
+}
+
+func (this *RawEncoder) EncodeBytes(data []byte) ([]byte, error) {
+	return data, nil
+}

pdf/core/encoding_test.go

@@ -0,0 +1,91 @@
+/*
+ * This file is subject to the terms and conditions defined in
+ * file 'LICENSE.md', which is part of this source code package.
+ */
+
+package core
+
+import (
+	"testing"
+
+	"github.com/unidoc/unidoc/common"
+)
+
+func init() {
+	common.SetLogger(common.ConsoleLogger{})
+}
+
+// Test flate encoding.
+func TestFlateEncoding(t *testing.T) {
+	rawStream := []byte("this is a dummy text with some \x01\x02\x03 binary data")
+
+	encoder := NewFlateEncoder()
+	encoder.Predictor = 1
+
+	encoded, err := encoder.EncodeBytes(rawStream)
+	if err != nil {
+		t.Errorf("Failed to encode data: %v", err)
+		return
+	}
+
+	decoded, err := encoder.DecodeBytes(encoded)
+	if err != nil {
+		t.Errorf("Failed to decode data: %v", err)
+		return
+	}
+
+	if !compareSlices(decoded, rawStream) {
+		t.Errorf("Slices not matching")
+		t.Errorf("Decoded (%d): % x", len(encoded), encoded)
+		t.Errorf("Raw     (%d): % x", len(rawStream), rawStream)
+		return
+	}
+}
+
+// Test LZW encoding.
+func TestLZWEncoding(t *testing.T) {
+	rawStream := []byte("this is a dummy text with some \x01\x02\x03 binary data")
+
+	encoder := NewLZWEncoder()
+	// Only supporitng early change 0 for encoding at the moment.
+	encoder.EarlyChange = 0
+
+	encoded, err := encoder.EncodeBytes(rawStream)
+	if err != nil {
+		t.Errorf("Failed to encode data: %v", err)
+		return
+	}
+
+	decoded, err := encoder.DecodeBytes(encoded)
+	if err != nil {
+		t.Errorf("Failed to decode data: %v", err)
+		return
+	}
+
+	if !compareSlices(decoded, rawStream) {
+		t.Errorf("Slices not matching")
+		t.Errorf("Decoded (%d): % x", len(encoded), encoded)
+		t.Errorf("Raw     (%d): % x", len(rawStream), rawStream)
+		return
+	}
+}
+
+// Test ASCII hex encoding.
+func TestASCIIHexEncoding(t *testing.T) {
+	byteData := []byte{0xDE, 0xAD, 0xBE, 0xEF}
+	expected := []byte("DE AD BE EF >")
+
+	encoder := NewASCIIHexEncoder()
+	encoded, err := encoder.EncodeBytes(byteData)
+	if err != nil {
+		t.Errorf("Failed to encode data: %v", err)
+		return
+	}
+
+	if !compareSlices(encoded, expected) {
+		t.Errorf("Slices not matching")
+		t.Errorf("Expected (%d): %s", len(expected), expected)
+		t.Errorf("Encoded  (%d): %s", len(encoded), encoded)
+		return
+	}
+}

pdf/core/stream.go

@@ -6,364 +6,17 @@
 package core
 
 import (
-	"bytes"
-	"compress/zlib"
-	"encoding/hex"
 	"fmt"
 
-	// The compress/lzw does not work, slightly different, need the following one:
-	"golang.org/x/image/tiff/lzw"
-
 	"github.com/unidoc/unidoc/common"
 )
 
-// Flate decoding.
+// Creates the encoder from the stream's dictionary.
-func decodeStreamFlateDecode(obj *PdfObjectStream) ([]byte, error) {
+func NewEncoderFromStream(streamObj *PdfObjectStream) (StreamEncoder, error) {
-	// Revamp this support to handle TIFF predictor (2).
+	filterObj, hasFilter := (*(streamObj.PdfObjectDictionary))["Filter"]
-	// Also handle more filter bytes and check
-	// BitsPerComponent.  Default value is 8, currently we are only
-	// supporting that one.
-	predictor := 1
-
-	decodeParams, hasDecodeParams := (*(obj.PdfObjectDictionary))["DecodeParms"].(*PdfObjectDictionary)
-	if hasDecodeParams {
-		common.Log.Debug("decode params: %s", decodeParams.String())
-		predictor = int(*((*decodeParams)["Predictor"].(*PdfObjectInteger)))
-
-		obits, hasbits := (*decodeParams)["BitsPerComponent"]
-		if hasbits {
-			pbits, ok := obits.(*PdfObjectInteger)
-			if !ok {
-				common.Log.Debug("ERROR: Invalid BitsPerComponent")
-				return nil, fmt.Errorf("Invalid BitsPerComponent")
-			}
-			if *pbits != 8 {
-				return nil, fmt.Errorf("Currently only 8 bits for flatedecode supported")
-			}
-		}
-	}
-	common.Log.Debug("FlateDecode")
-	common.Log.Debug("Predictor: %d", predictor)
-
-	bufReader := bytes.NewReader(obj.Stream)
-	r, err := zlib.NewReader(bufReader)
-	if err != nil {
-		common.Log.Debug("Decoding error %v\n", err)
-		common.Log.Debug("Stream (%d) % x", len(obj.Stream), obj.Stream)
-		return nil, err
-	}
-	defer r.Close()
-
-	var outBuf bytes.Buffer
-	outBuf.ReadFrom(r)
-	outData := outBuf.Bytes()
-
-	if hasDecodeParams && predictor != 1 {
-		if predictor == 2 { // TIFF encoding: Needs some tests.
-			common.Log.Debug("Tiff encoding")
-
-			columns, ok := (*decodeParams)["Columns"].(*PdfObjectInteger)
-			if !ok {
-				common.Log.Debug("ERROR: Predictor Column missing\n")
-				return nil, fmt.Errorf("Predictor column missing")
-			}
-
-			colors := 1
-			pcolors, hascolors := (*decodeParams)["Colors"].(*PdfObjectInteger)
-			if hascolors {
-				// Number of interleaved color components per sample
-				colors = int(*pcolors)
-			}
-			common.Log.Debug("colors: %d", colors)
-
-			rowLength := int(*columns) * colors
-			rows := len(outData) / rowLength
-			if len(outData)%rowLength != 0 {
-				common.Log.Debug("ERROR: TIFF encoding: Invalid row length...")
-				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
-			}
-
-			if rowLength%colors != 0 {
-				return nil, fmt.Errorf("Invalid row length (%d) for colors %d", rowLength, colors)
-			}
-			common.Log.Debug("inp outData (%d): % x", len(outData), outData)
-
-			pOutBuffer := bytes.NewBuffer(nil)
-
-			// 0-255  -255 255 ; 0-255=-255;
-			for i := 0; i < rows; i++ {
-				rowData := outData[rowLength*i : rowLength*(i+1)]
-				//common.Log.Debug("RowData before: % d", rowData)
-				// Predicts the same as the sample to the left.
-				// Interleaved by colors.
-				for j := colors; j < rowLength; j++ {
-					rowData[j] = byte(int(rowData[j]+rowData[j-colors]) % 256)
-				}
-				// GH: Appears that this is not working as expected...
-				//common.Log.Debug("RowData after: % d", rowData)
-
-				pOutBuffer.Write(rowData)
-			}
-			pOutData := pOutBuffer.Bytes()
-			common.Log.Debug("POutData (%d): % x", len(pOutData), pOutData)
-			return pOutData, nil
-		} else if predictor >= 10 && predictor <= 15 {
-			common.Log.Debug("PNG Encoding")
-			columns, ok := (*decodeParams)["Columns"].(*PdfObjectInteger)
-			if !ok {
-				common.Log.Debug("ERROR: Predictor Column missing\n")
-				return nil, fmt.Errorf("Predictor column missing")
-			}
-			rowLength := int(*columns + 1) // 1 byte to specify predictor algorithms per row.
-			rows := len(outData) / rowLength
-			if len(outData)%rowLength != 0 {
-				common.Log.Debug("ERROR: Invalid row length...")
-				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
-			}
-
-			pOutBuffer := bytes.NewBuffer(nil)
-
-			common.Log.Debug("Predictor columns: %d", columns)
-			common.Log.Debug("Length: %d / %d = %d rows", len(outData), rowLength, rows)
-			prevRowData := make([]byte, rowLength)
-			for i := 0; i < rowLength; i++ {
-				prevRowData[i] = 0
-			}
-
-			for i := 0; i < rows; i++ {
-				rowData := outData[rowLength*i : rowLength*(i+1)]
-
-				fb := rowData[0]
-				switch fb {
-				case 0:
-					// No prediction. (No operation).
-				case 1:
-					// Sub: Predicts the same as the sample to the left.
-					for j := 2; j < rowLength; j++ {
-						rowData[j] = byte(int(rowData[j]+rowData[j-1]) % 256)
-					}
-				case 2:
-					// Up: Predicts the same as the sample above
-					for j := 1; j < rowLength; j++ {
-						rowData[j] = byte(int(rowData[j]+prevRowData[j]) % 256)
-					}
-				default:
-					common.Log.Debug("ERROR: Invalid filter byte (%d)", fb)
-					return nil, fmt.Errorf("Invalid filter byte (%d)", fb)
-				}
-
-				for i := 0; i < rowLength; i++ {
-					prevRowData[i] = rowData[i]
-				}
-				pOutBuffer.Write(rowData[1:])
-			}
-			pOutData := pOutBuffer.Bytes()
-			return pOutData, nil
-		} else {
-			common.Log.Debug("ERROR: Unsupported predictor (%d)", predictor)
-			return nil, fmt.Errorf("Unsupported predictor (%d)", predictor)
-		}
-	}
-
-	return outData, nil
-}
-
-// LZW decoding.
-func decodeStreamLZWDecode(obj *PdfObjectStream) ([]byte, error) {
-	// Revamp this support to handle TIFF predictor (2).
-	// Also handle more filter bytes and check
-	// BitsPerComponent.  Default value is 8, currently we are only
-	// supporting that one.
-	predictor := 1
-
-	decodeParams, hasDecodeParams := (*(obj.PdfObjectDictionary))["DecodeParms"].(*PdfObjectDictionary)
-	if hasDecodeParams {
-		common.Log.Debug("decode params: %s", decodeParams.String())
-		predictor = int(*((*decodeParams)["Predictor"].(*PdfObjectInteger)))
-
-		obits, hasbits := (*decodeParams)["BitsPerComponent"]
-		if hasbits {
-			pbits, ok := obits.(*PdfObjectInteger)
-			if !ok {
-				common.Log.Debug("ERROR: Invalid BitsPerComponent")
-				return nil, fmt.Errorf("Invalid BitsPerComponent")
-			}
-			if *pbits != 8 {
-				return nil, fmt.Errorf("Currently only 8 bits for LZW supported")
-			}
-		}
-	}
-	common.Log.Debug("LZW Decoding")
-	common.Log.Debug("Predictor: %d", predictor)
-
-	var outBuf bytes.Buffer
-	bufReader := bytes.NewReader(obj.Stream)
-	r := lzw.NewReader(bufReader, lzw.MSB, 8)
-	defer r.Close()
-
-	_, err := outBuf.ReadFrom(r)
-	if err != nil {
-		return nil, err
-	}
-
-	outData := outBuf.Bytes()
-	common.Log.Debug(" IN: (%d) % x", len(obj.Stream), obj.Stream)
-	common.Log.Debug("OUT: (%d) % x", len(outData), outData)
-
-	if hasDecodeParams && predictor != 1 {
-		if predictor == 2 { // TIFF encoding: Needs some tests.
-			common.Log.Debug("Tiff encoding")
-
-			columns, ok := (*decodeParams)["Columns"].(*PdfObjectInteger)
-			if !ok {
-				common.Log.Debug("ERROR: Predictor Column missing\n")
-				return nil, fmt.Errorf("Predictor column missing")
-			}
-
-			colors := 1
-			pcolors, hascolors := (*decodeParams)["Colors"].(*PdfObjectInteger)
-			if hascolors {
-				// Number of interleaved color components per sample
-				colors = int(*pcolors)
-			}
-			common.Log.Debug("colors: %d", colors)
-
-			rowLength := int(*columns) * colors
-			rows := len(outData) / rowLength
-			if len(outData)%rowLength != 0 {
-				common.Log.Debug("ERROR: TIFF encoding: Invalid row length...")
-				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
-			}
-
-			if rowLength%colors != 0 {
-				return nil, fmt.Errorf("Invalid row length (%d) for colors %d", rowLength, colors)
-			}
-			common.Log.Debug("inp outData (%d): % x", len(outData), outData)
-
-			pOutBuffer := bytes.NewBuffer(nil)
-
-			// 0-255  -255 255 ; 0-255=-255;
-			for i := 0; i < rows; i++ {
-				rowData := outData[rowLength*i : rowLength*(i+1)]
-				//common.Log.Debug("RowData before: % d", rowData)
-				// Predicts the same as the sample to the left.
-				// Interleaved by colors.
-				for j := colors; j < rowLength; j++ {
-					rowData[j] = byte(int(rowData[j]+rowData[j-colors]) % 256)
-				}
-				// GH: Appears that this is not working as expected...
-				//common.Log.Debug("RowData after: % d", rowData)
-
-				pOutBuffer.Write(rowData)
-			}
-			pOutData := pOutBuffer.Bytes()
-			common.Log.Debug("POutData (%d): % x", len(pOutData), pOutData)
-			return pOutData, nil
-		} else if predictor >= 10 && predictor <= 15 {
-			common.Log.Debug("PNG Encoding")
-			columns, ok := (*decodeParams)["Columns"].(*PdfObjectInteger)
-			if !ok {
-				common.Log.Debug("ERROR: Predictor Column missing\n")
-				return nil, fmt.Errorf("Predictor column missing")
-			}
-			rowLength := int(*columns + 1) // 1 byte to specify predictor algorithms per row.
-			rows := len(outData) / rowLength
-			if len(outData)%rowLength != 0 {
-				common.Log.Debug("ERROR: Invalid row length...")
-				return nil, fmt.Errorf("Invalid row length (%d/%d)", len(outData), rowLength)
-			}
-
-			pOutBuffer := bytes.NewBuffer(nil)
-
-			common.Log.Debug("Predictor columns: %d", columns)
-			common.Log.Debug("Length: %d / %d = %d rows", len(outData), rowLength, rows)
-			prevRowData := make([]byte, rowLength)
-			for i := 0; i < rowLength; i++ {
-				prevRowData[i] = 0
-			}
-
-			for i := 0; i < rows; i++ {
-				rowData := outData[rowLength*i : rowLength*(i+1)]
-
-				fb := rowData[0]
-				switch fb {
-				case 0:
-					// No prediction. (No operation).
-				case 1:
-					// Sub: Predicts the same as the sample to the left.
-					for j := 2; j < rowLength; j++ {
-						rowData[j] = byte(int(rowData[j]+rowData[j-1]) % 256)
-					}
-				case 2:
-					// Up: Predicts the same as the sample above
-					for j := 1; j < rowLength; j++ {
-						rowData[j] = byte(int(rowData[j]+prevRowData[j]) % 256)
-					}
-				default:
-					common.Log.Debug("ERROR: Invalid filter byte (%d)", fb)
-					return nil, fmt.Errorf("Invalid filter byte (%d)", fb)
-				}
-
-				for i := 0; i < rowLength; i++ {
-					prevRowData[i] = rowData[i]
-				}
-				pOutBuffer.Write(rowData[1:])
-			}
-			pOutData := pOutBuffer.Bytes()
-			return pOutData, nil
-		} else {
-			common.Log.Debug("ERROR: Unsupported predictor (%d)", predictor)
-			return nil, fmt.Errorf("Unsupported predictor (%d)", predictor)
-		}
-	}
-
-	return outData, nil
-}
-
-// Flate decoding.
-func decodeStreamASCIIHexDecode(obj *PdfObjectStream) ([]byte, error) {
-	bufReader := bytes.NewReader(obj.Stream)
-	inb := []byte{}
-	for {
-		b, err := bufReader.ReadByte()
-		if err != nil {
-			return nil, err
-		}
-		if b == '>' {
-			break
-		}
-		if IsWhiteSpace(b) {
-			continue
-		}
-		if (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F') || (b >= '0' && b <= '9') {
-			inb = append(inb, b)
-		} else {
-			common.Log.Debug("ERROR: Invalid ascii hex character (%c)", b)
-			return nil, fmt.Errorf("Invalid ascii hex character (%c)", b)
-		}
-	}
-	if len(inb)%2 == 1 {
-		inb = append(inb, '0')
-	}
-	common.Log.Debug("Inbound %s", inb)
-	outb := make([]byte, hex.DecodedLen(len(inb)))
-	_, err := hex.Decode(outb, inb)
-	if err != nil {
-		return nil, err
-	}
-	return outb, nil
-}
-
-// Decodes the stream.
-// Supports FlateDecode, ASCIIHexDecode, LZW.
-func DecodeStream(obj *PdfObjectStream) ([]byte, error) {
-	common.Log.Debug("Decode stream")
-	common.Log.Debug("filter %s", (*obj).PdfObjectDictionary)
-
-	filterObj, hasFilter := (*(obj.PdfObjectDictionary))["Filter"]
 	if !hasFilter {
 		// No filter, return raw data back.
-		return obj.Stream, nil
+		return NewRawEncoder(), nil
 	}
 
 	// The filter should be a name or an array with a list of filter names.
@@ -385,13 +38,35 @@ func DecodeStream(obj *PdfObjectStream) ([]byte, error) {
 	}
 
 	if *method == "FlateDecode" {
-		return decodeStreamFlateDecode(obj)
+		return newFlateEncoderFromStream(streamObj)
 	} else if *method == "ASCIIHexDecode" {
-		return decodeStreamASCIIHexDecode(obj)
+		return NewASCIIHexEncoder(), nil
 	} else if *method == "LZWDecode" {
-		return decodeStreamLZWDecode(obj)
+		return newLZWEncoderFromStream(streamObj)
 	}
 
 	common.Log.Debug("ERROR: Unsupported encoding method!")
 	return nil, fmt.Errorf("Unsupported encoding method (%s)", *method)
 }
+
+// Decodes the stream.
+// Supports FlateDecode, ASCIIHexDecode, LZW.
+func DecodeStream(streamObj *PdfObjectStream) ([]byte, error) {
+	common.Log.Debug("Decode stream")
+
+	encoder, err := NewEncoderFromStream(streamObj)
+	if err != nil {
+		common.Log.Debug("Stream decoding failed: %v", err)
+		return nil, err
+	}
+
+	common.Log.Debug("Encoder: %+v\n", encoder)
+
+	decoded, err := encoder.DecodeStream(streamObj)
+	if err != nil {
+		common.Log.Debug("Stream decoding failed: %v", err)
+		return nil, err
+	}
+
+	return decoded, nil
+}

pdf/model/page.go

@@ -651,7 +651,56 @@ func (this *PdfPage) AddContentStreamByString(contentStr string) {
 		contArray = append(contArray, &stream)
 		this.Contents = &contArray
 	}
+}
 
+// Set the content streams based on a string array.  Will make 1 object stream
+// for each string and reference from the page Contents.  Each stream will be
+// encoded using the encoding specified by the StreamEncoder, if empty, will
+// use identity encoding (raw data).
+func (this *PdfPage) SetContentStreams(cStreams []string, encoder StreamEncoder) error {
+	if len(cStreams) == 0 {
+		this.Contents = nil
+		return nil
+	}
+
+	// If encoding is not set, use default raw encoder.
+	if encoder == nil {
+		encoder = NewRawEncoder()
+	}
+
+	streamObjs := []*PdfObjectStream{}
+	for _, cStream := range cStreams {
+		stream := &PdfObjectStream{}
+
+		// Make a new stream dict based on the encoding parameters.
+		sDict := encoder.MakeEncodingDict()
+
+		encoded, err := encoder.EncodeBytes([]byte(cStream))
+		if err != nil {
+			return err
+		}
+
+		(*sDict)["Length"] = MakeInteger(int64(len(encoded)))
+
+		stream.PdfObjectDictionary = sDict
+		stream.Stream = []byte(encoded)
+
+		streamObjs = append(streamObjs, stream)
+	}
+
+	// Set the page contents.
+	// Point directly to the object stream if only one, or embed in an array.
+	if len(streamObjs) == 1 {
+		this.Contents = streamObjs[0]
+	} else {
+		contArray := PdfObjectArray{}
+		for _, streamObj := range streamObjs {
+			contArray = append(contArray, streamObj)
+		}
+		this.Contents = &contArray
+	}
+
+	return nil
 }
 
 func getContentStreamAsString(cstreamObj PdfObject) (string, error) {