mirror of
https://github.com/sjwhitworth/golearn.git
synced 2025-04-25 13:48:49 +08:00
neural: stop-gap support for neural networks
This commit is contained in:
Richard Townsend
parent
29b3f06566
commit
f9c1e24e5b
@ -8,7 +8,7 @@ import (
|
||||
func TestFloatAttributeSysVal(t *testing.T) {
|
||||
Convey("Given some float", t, func() {
|
||||
x := "1.21"
|
||||
attr := NewFloatAttribute()
|
||||
attr := NewFloatAttribute("")
|
||||
Convey("When the float gets packed", func() {
|
||||
packed := attr.GetSysValFromString(x)
|
||||
Convey("And then unpacked", func() {
|
||||
|
||||
@ -107,7 +107,7 @@ func ParseCSVSniffAttributeTypes(filepath string, hasHeaders bool) []Attribute {
|
||||
panic(err)
|
||||
}
|
||||
if matched {
|
||||
attrs = append(attrs, NewFloatAttribute())
|
||||
attrs = append(attrs, NewFloatAttribute(""))
|
||||
} else {
|
||||
attrs = append(attrs, new(CategoricalAttribute))
|
||||
}
|
||||
|
||||
@ -14,8 +14,8 @@ type FloatAttribute struct {
|
||||
|
||||
// NewFloatAttribute returns a new FloatAttribute with a default
|
||||
// precision of 2 decimal places
|
||||
func NewFloatAttribute() *FloatAttribute {
|
||||
return &FloatAttribute{"", 2}
|
||||
func NewFloatAttribute(name string) *FloatAttribute {
|
||||
return &FloatAttribute{name, 2}
|
||||
}
|
||||
|
||||
// Compatable checks whether this FloatAttribute can be ponded with another
|
||||
|
||||
@ -63,8 +63,7 @@ func main() {
|
||||
|
||||
// Let's create some attributes
|
||||
attrs := make([]base.Attribute, 2)
|
||||
attrs[0] = base.NewFloatAttribute()
|
||||
attrs[0].SetName("Arbitrary Float Quantity")
|
||||
attrs[0] = base.NewFloatAttribute("Arbitrary Float Quantity")
|
||||
attrs[1] = new(base.CategoricalAttribute)
|
||||
attrs[1].SetName("Class")
|
||||
// Insert a standard class
|
||||
|
||||
19
neural/funcs.go
Normal file
19
neural/funcs.go
Normal file
@ -0,0 +1,19 @@
|
||||
package neural
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// SigmoidForward function does S(t) = \frac{1}{1 + e^{-t}}.
|
||||
//
|
||||
// See http://en.wikipedia.org/wiki/Sigmoid_function
|
||||
var Sigmoid NeuralFunction = NeuralFunction{
|
||||
func(v float64) float64 { return 1.0 / (1.0 + math.Exp(-v)) },
|
||||
func(v float64) float64 { return v * (1 - v) },
|
||||
}
|
||||
|
||||
// LinearFunction doesn't modify the value
|
||||
var Linear NeuralFunction = NeuralFunction{
|
||||
func(v float64) float64 { return v },
|
||||
func(v float64) float64 { return 1.0 },
|
||||
}
|
||||
345
neural/layered.go
Normal file
345
neural/layered.go
Normal file
@ -0,0 +1,345 @@
|
||||
package neural
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/gonum/matrix/mat64"
|
||||
"github.com/sjwhitworth/golearn/base"
|
||||
"github.com/sjwhitworth/golearn/filters"
|
||||
"math"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
// MultiLayerNet creates a new Network which is conceptually
|
||||
// organised into layers, zero or more of which are hidden.
|
||||
//
|
||||
// Within each layer, no neurons are connected.
|
||||
//
|
||||
// No neurons in a given layer are connected with any neurons
|
||||
// in a previous layer.
|
||||
//
|
||||
// Neurons can only be connected to neurons in the layer above.
|
||||
type MultiLayerNet struct {
|
||||
network *Network
|
||||
attrs map[base.Attribute]int
|
||||
layers []int
|
||||
classAttrOffset int
|
||||
classAttrCount int
|
||||
Convergence float64
|
||||
MaxIterations int
|
||||
LearningRate float64
|
||||
}
|
||||
|
||||
// NewMultiLayerNet returns an underlying
|
||||
// Network conceptuallyorganised into layers
|
||||
//
|
||||
// Layers variable = slice of integers representing
|
||||
// node count at each layer.
|
||||
func NewMultiLayerNet(layers []int) *MultiLayerNet {
|
||||
return &MultiLayerNet{
|
||||
nil,
|
||||
make(map[base.Attribute]int),
|
||||
layers,
|
||||
0,
|
||||
0,
|
||||
0.001,
|
||||
500,
|
||||
0.90,
|
||||
}
|
||||
}
|
||||
|
||||
// String returns a human-readable summary of this network.
|
||||
func (m *MultiLayerNet) String() string {
|
||||
return fmt.Sprintf("MultiLayerNet(%v, %v, %.2f, %.2f, %d", m.layers, m.network, m.Convergence, m.LearningRate, m.MaxIterations)
|
||||
}
|
||||
|
||||
func (m *MultiLayerNet) convertToFloatInsts(X base.FixedDataGrid) base.FixedDataGrid {
|
||||
|
||||
// Make sure everything's a FloatAttribute
|
||||
fFilt := filters.NewFloatConvertFilter()
|
||||
for _, a := range X.AllAttributes() {
|
||||
fFilt.AddAttribute(a)
|
||||
}
|
||||
fFilt.Train()
|
||||
insts := base.NewLazilyFilteredInstances(X, fFilt)
|
||||
return insts
|
||||
}
|
||||
|
||||
// Predict uses the underlying network to produce predictions for the
|
||||
// class variables of X.
|
||||
//
|
||||
// Can only predict one CategoricalAttribute at a time, or up to n
|
||||
// FloatAttributes. Set or unset ClassAttributes to work around this
|
||||
// limitation.
|
||||
func (m *MultiLayerNet) Predict(X base.FixedDataGrid) base.FixedDataGrid {
|
||||
|
||||
// Create the return vector
|
||||
ret := base.GeneratePredictionVector(X)
|
||||
|
||||
// Make sure everything's a FloatAttribute
|
||||
insts := m.convertToFloatInsts(X)
|
||||
|
||||
// Get the input/output Attributes
|
||||
inputAttrs := base.NonClassAttributes(insts)
|
||||
outputAttrs := ret.AllClassAttributes()
|
||||
|
||||
// Compute layers
|
||||
layers := 2 + len(m.layers)
|
||||
|
||||
// Check that we're operating in a singular mode
|
||||
floatMode := 0
|
||||
categoricalMode := 0
|
||||
for _, a := range outputAttrs {
|
||||
if _, ok := a.(*base.CategoricalAttribute); ok {
|
||||
categoricalMode++
|
||||
} else if _, ok := a.(*base.FloatAttribute); ok {
|
||||
floatMode++
|
||||
} else {
|
||||
panic("Unsupported output Attribute type!")
|
||||
}
|
||||
}
|
||||
|
||||
if floatMode > 0 && categoricalMode > 0 {
|
||||
panic("Can't predict a mix of float and categorical Attributes")
|
||||
} else if categoricalMode > 1 {
|
||||
panic("Can't predict more than one categorical class Attribute")
|
||||
}
|
||||
|
||||
// Create the activation vector
|
||||
a := mat64.NewDense(m.network.size, 1, make([]float64, m.network.size))
|
||||
|
||||
// Resolve the input AttributeSpecs
|
||||
inputAs := base.ResolveAttributes(insts, inputAttrs)
|
||||
|
||||
// Resolve the output Attributespecs
|
||||
outputAs := base.ResolveAttributes(ret, outputAttrs)
|
||||
|
||||
// Map over each input row
|
||||
insts.MapOverRows(inputAs, func(row [][]byte, rc int) (bool, error) {
|
||||
// Clear the activation vector
|
||||
for i := 0; i < m.network.size; i++ {
|
||||
a.Set(i, 0, 0.0)
|
||||
}
|
||||
// Build the activation vector
|
||||
for i, vb := range row {
|
||||
if cIndex, ok := m.attrs[inputAs[i].GetAttribute()]; !ok {
|
||||
panic("Can't resolve the Attribute!")
|
||||
} else {
|
||||
a.Set(cIndex, 0, base.UnpackBytesToFloat(vb))
|
||||
}
|
||||
}
|
||||
// Robots, activate!
|
||||
m.network.Activate(a, layers)
|
||||
|
||||
// Decide which class to set
|
||||
if floatMode > 0 {
|
||||
for _, as := range outputAs {
|
||||
cIndex := m.attrs[as.GetAttribute()]
|
||||
ret.Set(as, rc, base.PackFloatToBytes(a.At(cIndex, 0)))
|
||||
}
|
||||
} else {
|
||||
maxIndex := 0
|
||||
maxVal := 0.0
|
||||
for i := m.classAttrOffset; i < m.classAttrOffset+m.classAttrCount; i++ {
|
||||
val := a.At(i, 0)
|
||||
if val > maxVal {
|
||||
maxIndex = i
|
||||
maxVal = val
|
||||
}
|
||||
}
|
||||
maxIndex -= m.classAttrOffset
|
||||
ret.Set(outputAs[0], rc, base.PackU64ToBytes(uint64(maxIndex)))
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
|
||||
return ret
|
||||
|
||||
}
|
||||
|
||||
// Fit trains the neural network on the given fixed datagrid.
|
||||
//
|
||||
// Training stops when the mean-squared error acheived is less
|
||||
// than the Convergence value, or when back-propagation has occured
|
||||
// more times than the value set by MaxIterations.
|
||||
func (m *MultiLayerNet) Fit(X base.FixedDataGrid) {
|
||||
|
||||
// Make sure everything's a FloatAttribute
|
||||
insts := m.convertToFloatInsts(X)
|
||||
|
||||
// The size of the first layer is the number of things
|
||||
// in the revised instances which aren't class Attributes
|
||||
inputAttrsVec := base.NonClassAttributes(insts)
|
||||
|
||||
// The size of the output layer is the number of things
|
||||
// in the revised instances which are class Attributes
|
||||
classAttrsVec := insts.AllClassAttributes()
|
||||
|
||||
// The total number of layers is input layer + output layer
|
||||
// plus number of layers specified
|
||||
totalLayers := 2 + len(m.layers)
|
||||
|
||||
// The size is then augmented by the number of nodes
|
||||
// in the centre
|
||||
size := len(inputAttrsVec)
|
||||
size += len(classAttrsVec)
|
||||
hiddenSize := 0
|
||||
for _, a := range m.layers {
|
||||
size += a
|
||||
hiddenSize += a
|
||||
}
|
||||
|
||||
// Enumerate the Attributes
|
||||
trainingAttrs := make(map[base.Attribute]int)
|
||||
classAttrs := make(map[base.Attribute]int)
|
||||
attrCounter := 0
|
||||
for i, a := range inputAttrsVec {
|
||||
attrCounter = i
|
||||
m.attrs[a] = attrCounter
|
||||
trainingAttrs[a] = attrCounter
|
||||
}
|
||||
m.classAttrOffset = attrCounter + 1
|
||||
for _, a := range classAttrsVec {
|
||||
attrCounter++
|
||||
m.attrs[a] = attrCounter + hiddenSize
|
||||
classAttrs[a] = attrCounter + hiddenSize
|
||||
m.classAttrCount++
|
||||
}
|
||||
|
||||
// Create the underlying Network
|
||||
m.network = NewNetwork(size, len(inputAttrsVec), Sigmoid)
|
||||
|
||||
// Initialise inter-hidden layer weights and biases to small random values
|
||||
layerOffset := len(inputAttrsVec)
|
||||
for i := 0; i < len(m.layers)-1; i++ {
|
||||
// Get the size of this layer
|
||||
thisLayerSize := m.layers[i]
|
||||
// Next layer size
|
||||
nextLayerSize := m.layers[i+1]
|
||||
// For every node in this layer
|
||||
for j := 1; j <= thisLayerSize; j++ {
|
||||
// Compute the offset
|
||||
nodeOffset1 := layerOffset + j
|
||||
// For every node in the next layer
|
||||
for k := 1; k <= nextLayerSize; k++ {
|
||||
// Compute offset
|
||||
nodeOffset2 := layerOffset + thisLayerSize + k
|
||||
// Set weight randomly
|
||||
m.network.SetWeight(nodeOffset1, nodeOffset2, rand.NormFloat64()*0.1)
|
||||
}
|
||||
}
|
||||
layerOffset += thisLayerSize
|
||||
}
|
||||
|
||||
// Initialise biases with each hidden layer
|
||||
layerOffset = len(inputAttrsVec)
|
||||
for _, l := range m.layers {
|
||||
for j := 1; j <= l; j++ {
|
||||
nodeOffset := layerOffset + j
|
||||
m.network.SetBias(nodeOffset, rand.NormFloat64()*0.1)
|
||||
}
|
||||
layerOffset += l
|
||||
}
|
||||
|
||||
// Initialise biases for output layer
|
||||
for i := 0; i < len(classAttrsVec); i++ {
|
||||
nodeOffset := layerOffset + i
|
||||
m.network.SetBias(nodeOffset, rand.NormFloat64()*0.1)
|
||||
}
|
||||
|
||||
// Connect final hidden layer with the output layer
|
||||
layerOffset = len(inputAttrsVec)
|
||||
for i, l := range m.layers {
|
||||
if i == len(m.layers)-1 {
|
||||
for j := 1; j <= l; j++ {
|
||||
nodeOffset1 := layerOffset + j
|
||||
for k := 1; k <= len(classAttrsVec); k++ {
|
||||
nodeOffset2 := layerOffset + l + k
|
||||
m.network.SetWeight(nodeOffset1, nodeOffset2, rand.NormFloat64()*0.1)
|
||||
}
|
||||
}
|
||||
}
|
||||
layerOffset += l
|
||||
}
|
||||
|
||||
// Connect input layer with first hidden layer (or output layer
|
||||
for i := 1; i <= len(inputAttrsVec); i++ {
|
||||
nextLayerLen := 0
|
||||
if len(m.layers) > 0 {
|
||||
nextLayerLen = m.layers[0]
|
||||
} else {
|
||||
nextLayerLen = len(classAttrsVec)
|
||||
}
|
||||
for j := 1; j <= nextLayerLen; j++ {
|
||||
nodeOffset := len(inputAttrsVec) + j
|
||||
v := rand.NormFloat64() * 0.1
|
||||
m.network.SetWeight(i, nodeOffset, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Create the training activation vector
|
||||
trainVec := mat64.NewDense(size, 1, make([]float64, size))
|
||||
// Create the error vector
|
||||
errVec := mat64.NewDense(size, 1, make([]float64, size))
|
||||
|
||||
// Resolve training AttributeSpecs
|
||||
trainAs := base.ResolveAllAttributes(insts)
|
||||
|
||||
// Feed-forward, compute error and update for each training example
|
||||
// until convergence (what's that)
|
||||
for iteration := 0; iteration < m.MaxIterations; iteration++ {
|
||||
totalError := 0.0
|
||||
maxRow := 0
|
||||
insts.MapOverRows(trainAs, func(row [][]byte, i int) (bool, error) {
|
||||
|
||||
maxRow = i
|
||||
// Clear vectors
|
||||
for i := 0; i < size; i++ {
|
||||
trainVec.Set(i, 0, 0.0)
|
||||
errVec.Set(i, 0, 0.0)
|
||||
}
|
||||
|
||||
// Build vectors
|
||||
for i, vb := range row {
|
||||
v := base.UnpackBytesToFloat(vb)
|
||||
if attrIndex, ok := trainingAttrs[trainAs[i].GetAttribute()]; ok {
|
||||
// Add to Activation vector
|
||||
trainVec.Set(attrIndex, 0, v)
|
||||
} else if attrIndex, ok := classAttrs[trainAs[i].GetAttribute()]; ok {
|
||||
// Set to error vector
|
||||
errVec.Set(attrIndex, 0, v)
|
||||
} else {
|
||||
panic("Should be able to find this Attribute!")
|
||||
}
|
||||
}
|
||||
|
||||
// Activate the network
|
||||
m.network.Activate(trainVec, totalLayers-1)
|
||||
|
||||
// Compute the error
|
||||
for a := range classAttrs {
|
||||
cIndex := classAttrs[a]
|
||||
errVec.Set(cIndex, 0, errVec.At(cIndex, 0)-trainVec.At(cIndex, 0))
|
||||
}
|
||||
|
||||
// Update total error
|
||||
totalError += math.Abs(errVec.Sum())
|
||||
|
||||
// Back-propagate the error
|
||||
b := m.network.Error(trainVec, errVec, totalLayers)
|
||||
|
||||
// Update the weights
|
||||
m.network.UpdateWeights(trainVec, b, m.LearningRate)
|
||||
|
||||
// Update the biases
|
||||
m.network.UpdateBias(b, m.LearningRate)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
totalError /= float64(maxRow)
|
||||
// If we've converged, no need to carry on
|
||||
if totalError < m.Convergence {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
166
neural/layered_test.go
Normal file
166
neural/layered_test.go
Normal file
@ -0,0 +1,166 @@
|
||||
package neural
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/gonum/matrix/mat64"
|
||||
"github.com/sjwhitworth/golearn/base"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLayerStructureNoHidden(t *testing.T) {
|
||||
|
||||
Convey("Creating a network...", t, func() {
|
||||
XORData, err := base.ParseCSVToInstances("xor.csv", false)
|
||||
So(err, ShouldEqual, nil)
|
||||
fmt.Println(XORData)
|
||||
Convey("Create a MultiLayerNet with no layers...", func() {
|
||||
net := NewMultiLayerNet(make([]int, 0))
|
||||
net.MaxIterations = 0
|
||||
net.Fit(XORData)
|
||||
Convey("The network should be the right size...", func() {
|
||||
So(net.network.size, ShouldEqual, 3)
|
||||
})
|
||||
Convey("The right nodes should be connected in the network...", func() {
|
||||
So(net.network.GetWeight(1, 1), ShouldAlmostEqual, 1.000)
|
||||
So(net.network.GetWeight(2, 2), ShouldAlmostEqual, 1.000)
|
||||
So(net.network.GetWeight(3, 3), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(1, 3), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(2, 3), ShouldNotAlmostEqual, 0.000)
|
||||
})
|
||||
})
|
||||
Convey("Create a multilayer net with two hidden layers...", func() {
|
||||
net := NewMultiLayerNet([]int{3, 2})
|
||||
net.MaxIterations = 0
|
||||
net.Fit(XORData)
|
||||
Convey("The network should be the right size...", func() {
|
||||
So(net.network.size, ShouldEqual, 8)
|
||||
})
|
||||
Convey("The right nodes should be connected in the network...", func() {
|
||||
So(net.network.GetWeight(1, 1), ShouldAlmostEqual, 1.000)
|
||||
So(net.network.GetWeight(2, 2), ShouldAlmostEqual, 1.000)
|
||||
for i := 3; i <= 8; i++ {
|
||||
So(net.network.GetWeight(i, i), ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
for i := 1; i <= 2; i++ {
|
||||
for j := 3; j <= 5; j++ {
|
||||
So(net.network.GetWeight(i, j), ShouldNotAlmostEqual, 0.000)
|
||||
}
|
||||
}
|
||||
for i := 3; i <= 5; i++ {
|
||||
for j := 6; j <= 7; j++ {
|
||||
So(net.network.GetWeight(i, j), ShouldNotAlmostEqual, 0.000)
|
||||
}
|
||||
}
|
||||
for i := 6; i <= 7; i++ {
|
||||
So(net.network.GetWeight(i, 8), ShouldNotAlmostEqual, 0.000)
|
||||
}
|
||||
for i := 8; i > 0; i-- {
|
||||
for j := i - 1; j > 0; j-- {
|
||||
So(net.network.GetWeight(i, j), ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
Convey("Create a MultiLayerNet with 1 hidden layer...", func() {
|
||||
net := NewMultiLayerNet([]int{3})
|
||||
net.LearningRate = 0.9
|
||||
net.MaxIterations = 0
|
||||
net.Fit(XORData)
|
||||
|
||||
Convey("The network should be the right size...", func() {
|
||||
So(net.network.size, ShouldEqual, 6)
|
||||
})
|
||||
|
||||
Convey("The right nodes should be connected in the network...", func() {
|
||||
|
||||
fmt.Println(net.network)
|
||||
So(net.network.GetWeight(1, 1), ShouldAlmostEqual, 1.000)
|
||||
So(net.network.GetWeight(2, 2), ShouldAlmostEqual, 1.000)
|
||||
So(net.network.GetWeight(1, 3), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(1, 4), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(1, 5), ShouldNotAlmostEqual, 0.000)
|
||||
|
||||
So(net.network.GetWeight(2, 3), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(2, 4), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(2, 5), ShouldNotAlmostEqual, 0.000)
|
||||
|
||||
So(net.network.GetWeight(3, 3), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(3, 4), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(3, 5), ShouldAlmostEqual, 0.000)
|
||||
|
||||
So(net.network.GetWeight(4, 4), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(4, 3), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(4, 5), ShouldAlmostEqual, 0.000)
|
||||
|
||||
So(net.network.GetWeight(5, 5), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(5, 3), ShouldAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(5, 4), ShouldAlmostEqual, 0.000)
|
||||
|
||||
So(net.network.GetWeight(3, 6), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(4, 6), ShouldNotAlmostEqual, 0.000)
|
||||
So(net.network.GetWeight(5, 6), ShouldNotAlmostEqual, 0.000)
|
||||
|
||||
for i := 1; i <= 6; i++ {
|
||||
So(net.network.GetWeight(6, i), ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func TestLayeredXOR(t *testing.T) {
|
||||
|
||||
Convey("Given an XOR dataset...", t, func() {
|
||||
|
||||
XORData, err := base.ParseCSVToInstances("xor.csv", false)
|
||||
So(err, ShouldEqual, nil)
|
||||
|
||||
fmt.Println(XORData)
|
||||
net := NewMultiLayerNet([]int{3})
|
||||
net.MaxIterations = 20000
|
||||
net.Fit(XORData)
|
||||
|
||||
Convey("After running for 20000 iterations, should have some predictive power...", func() {
|
||||
|
||||
Convey("The right nodes should be connected in the network...", func() {
|
||||
|
||||
fmt.Println(net.network)
|
||||
So(net.network.GetWeight(1, 1), ShouldAlmostEqual, 1.000)
|
||||
So(net.network.GetWeight(2, 2), ShouldAlmostEqual, 1.000)
|
||||
|
||||
for i := 1; i <= 6; i++ {
|
||||
So(net.network.GetWeight(6, i), ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
|
||||
})
|
||||
out := mat64.NewDense(6, 1, []float64{1.0, 0.0, 0.0, 0.0, 0.0, 0.0})
|
||||
net.network.Activate(out, 2)
|
||||
fmt.Println(out)
|
||||
So(out.At(5, 0), ShouldAlmostEqual, 1.0, 0.1)
|
||||
|
||||
Convey("And Predict() should do OK too...", func() {
|
||||
|
||||
pred := net.Predict(XORData)
|
||||
|
||||
for _, a := range pred.AllAttributes() {
|
||||
af, ok := a.(*base.FloatAttribute)
|
||||
if !ok {
|
||||
panic("All of these should be FloatAttributes!")
|
||||
}
|
||||
af.Precision = 1
|
||||
}
|
||||
|
||||
So(base.GetClass(pred, 0), ShouldEqual, "0.0")
|
||||
So(base.GetClass(pred, 1), ShouldEqual, "1.0")
|
||||
So(base.GetClass(pred, 2), ShouldEqual, "1.0")
|
||||
So(base.GetClass(pred, 3), ShouldEqual, "0.0")
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
222
neural/network.go
Normal file
222
neural/network.go
Normal file
@ -0,0 +1,222 @@
|
||||
package neural
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/gonum/blas/cblas"
|
||||
"github.com/gonum/matrix/mat64"
|
||||
"math"
|
||||
)
|
||||
|
||||
// Network represents the most general neural network possible
|
||||
// Weights are stored in a dense matrix, each can have its own
|
||||
// NeuralFunction.
|
||||
type Network struct {
|
||||
origWeights *mat64.Dense
|
||||
weights *mat64.Dense // n * n
|
||||
biases []float64 // n for each neuron
|
||||
funcs []NeuralFunction // for each neuron
|
||||
size int
|
||||
input int
|
||||
}
|
||||
|
||||
// NewNetwork creates a new Network containing size neurons,
|
||||
// with a certain number dedicated to input, and a pre-defined
|
||||
// neural function applied to the rest.
|
||||
//
|
||||
// Input nodes are set to have a Linear NeuralFunction and are
|
||||
// connected to themselves for propagation.
|
||||
func NewNetwork(size int, input int, f NeuralFunction) *Network {
|
||||
ret := new(Network)
|
||||
ret.weights = mat64.NewDense(size, size, make([]float64, size*size))
|
||||
ret.biases = make([]float64, size)
|
||||
ret.funcs = make([]NeuralFunction, size)
|
||||
ret.size = size
|
||||
ret.input = input
|
||||
for i := range ret.funcs {
|
||||
ret.funcs[i] = f
|
||||
}
|
||||
for i := 0; i < input; i++ {
|
||||
ret.funcs[i] = Linear
|
||||
ret.SetWeight(i+1, i+1, 1.0)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// String gets a human-readable representation of this network.
|
||||
func (n *Network) String() string {
|
||||
var buf bytes.Buffer
|
||||
var biases bytes.Buffer
|
||||
|
||||
for i := 0; i < n.size; i++ {
|
||||
for j := 0; j < n.size; j++ {
|
||||
v := n.weights.At(j, i)
|
||||
if math.Abs(v) > 0 {
|
||||
buf.WriteString(fmt.Sprintf("\t(%d %d %.2f)\n", i+1, j+1, v))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range n.biases {
|
||||
biases.WriteString(fmt.Sprintf(" %.2f", v))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("Network(%d, %s, %s)", n.size, biases.String(), buf.String())
|
||||
}
|
||||
|
||||
// GetWeight returns the weight between a given source and
|
||||
// target neuron (counted from 1).
|
||||
func (n *Network) GetWeight(src, target int) float64 {
|
||||
src--
|
||||
target--
|
||||
return n.weights.At(target, src)
|
||||
}
|
||||
|
||||
// SetWeight sets the weight between a given source and
|
||||
// target neuron (counted from 1).
|
||||
func (n *Network) SetWeight(src, target int, v float64) {
|
||||
src--
|
||||
target--
|
||||
n.weights.Set(target, src, v)
|
||||
}
|
||||
|
||||
// SetBias sets the bias at a given neuron (counted from 1).
|
||||
func (n *Network) SetBias(node int, v float64) {
|
||||
if node <= n.input {
|
||||
return
|
||||
}
|
||||
node--
|
||||
n.biases[node] = v
|
||||
}
|
||||
|
||||
// GetBias returns the bias at a given neuron (counted from 1).
|
||||
func (n *Network) GetBias(node int) float64 {
|
||||
node--
|
||||
return n.biases[node]
|
||||
}
|
||||
|
||||
// Activate propagates the given input matrix (with) across the network
|
||||
// a certain number of times (up to maxIterations).
|
||||
//
|
||||
// The with matrix should be size * size elements, with only the values
|
||||
// of input neurons set (everything else should be zero).
|
||||
//
|
||||
// If the network is conceptually organised into layers, maxIterations
|
||||
// should be set to the number of layers.
|
||||
//
|
||||
// This function overwrites whatever's stored in its first argument.
|
||||
func (n *Network) Activate(with *mat64.Dense, maxIterations int) {
|
||||
|
||||
// Add bias and feed to activation
|
||||
biasFunc := func(r, c int, v float64) float64 {
|
||||
return v + n.biases[r]
|
||||
}
|
||||
activFunc := func(r, c int, v float64) float64 {
|
||||
return n.funcs[r].Forward(v)
|
||||
}
|
||||
|
||||
tmp := new(mat64.Dense)
|
||||
tmp.Clone(with)
|
||||
|
||||
mat64.Register(cblas.Blas{})
|
||||
|
||||
// Main loop
|
||||
for i := 0; i < maxIterations; i++ {
|
||||
with.Mul(n.weights, with)
|
||||
with.Apply(biasFunc, with)
|
||||
with.Apply(activFunc, with)
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateWeights takes an output size * 1 output vector and a size * 1
|
||||
// back-propagated error vector, as well as a learnRate and updates
|
||||
// the internal weights matrix.
|
||||
func (n *Network) UpdateWeights(out, err *mat64.Dense, learnRate float64) {
|
||||
|
||||
if n.origWeights == nil {
|
||||
n.origWeights = mat64.DenseCopyOf(n.weights)
|
||||
}
|
||||
|
||||
// Multiply that by the learning rate
|
||||
mulFunc := func(target, source int, v float64) float64 {
|
||||
if target == source {
|
||||
return v
|
||||
}
|
||||
if math.Abs(n.origWeights.At(target, source)) > 0.005 {
|
||||
return v + learnRate*out.At(source, 0)*err.At(target, 0)
|
||||
}
|
||||
return 0.00
|
||||
}
|
||||
|
||||
// Add that to the weights
|
||||
n.weights.Apply(mulFunc, n.weights)
|
||||
}
|
||||
|
||||
// UpdateBias computes B = B + l.E and updates the bias weights
|
||||
// from a size * 1 back-propagated error vector.
|
||||
func (n *Network) UpdateBias(err *mat64.Dense, learnRate float64) {
|
||||
|
||||
for i, b := range n.biases {
|
||||
if i < n.input {
|
||||
continue
|
||||
}
|
||||
n.biases[i] = b + err.At(i, 0)*learnRate
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Error computes the back-propagation error from a given size * 1 output
|
||||
// vector and a size * 1 error vector for a given number of iterations.
|
||||
//
|
||||
// outArg should be the response from Activate.
|
||||
//
|
||||
// errArg should be the difference between the output neuron's output and
|
||||
// that expected, and should be zero everywhere else.
|
||||
//
|
||||
// If the network is conceptually organised into n layers, maxIterations
|
||||
// should be set to n.
|
||||
func (n *Network) Error(outArg, errArg *mat64.Dense, maxIterations int) *mat64.Dense {
|
||||
|
||||
// Copy the arguments
|
||||
out := mat64.DenseCopyOf(outArg)
|
||||
err := mat64.DenseCopyOf(errArg)
|
||||
|
||||
// err should be the difference between observed and expected
|
||||
// for observation nodes only (everything else should be zero)
|
||||
|
||||
// Allocate output vector
|
||||
outRows, outCols := out.Dims()
|
||||
if outCols != 1 {
|
||||
panic("Unsupported output size")
|
||||
}
|
||||
|
||||
ret := mat64.NewDense(outRows, 1, make([]float64, outRows))
|
||||
|
||||
// Do differential calculation
|
||||
diffFunc := func(r, c int, v float64) float64 {
|
||||
return n.funcs[r].Backward(v)
|
||||
}
|
||||
out.Apply(diffFunc, out)
|
||||
|
||||
// Transpose weights matrix
|
||||
reverseWeights := mat64.DenseCopyOf(n.weights)
|
||||
reverseWeights.TCopy(n.weights)
|
||||
|
||||
// We only need a certain number of passes
|
||||
for i := 0; i < maxIterations; i++ {
|
||||
|
||||
// Element-wise multiply errors and derivatives
|
||||
err.MulElem(err, out)
|
||||
|
||||
// Add the accumulated error
|
||||
ret.Add(ret, err)
|
||||
|
||||
if i != maxIterations-1 {
|
||||
// Feed the errors backwards through the network
|
||||
err.Mul(reverseWeights, err)
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
|
||||
}
|
||||
133
neural/network_test.go
Normal file
133
neural/network_test.go
Normal file
@ -0,0 +1,133 @@
|
||||
package neural
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/gonum/matrix/mat64"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNetworkWith1Layer(t *testing.T) {
|
||||
|
||||
Convey("Given the Network from Han and Kamber (p 334)...", t, func() {
|
||||
// Contains 6 nodes, 3 input nodes and uses Sigmoid
|
||||
n := NewNetwork(6, 3, Sigmoid)
|
||||
|
||||
// Set the weights
|
||||
n.SetWeight(1, 4, 0.2)
|
||||
n.SetWeight(1, 5, -0.3)
|
||||
n.SetWeight(2, 4, 0.4)
|
||||
n.SetWeight(2, 5, 0.1)
|
||||
n.SetWeight(3, 4, -0.5)
|
||||
n.SetWeight(3, 5, 0.2)
|
||||
n.SetWeight(4, 6, -0.3)
|
||||
n.SetWeight(5, 6, -0.2)
|
||||
|
||||
// Set the biases
|
||||
n.SetBias(4, -0.4)
|
||||
n.SetBias(5, 0.2)
|
||||
n.SetBias(6, 0.1)
|
||||
|
||||
// Create the Activation vector
|
||||
// NewDense is rows then columns
|
||||
a := mat64.NewDense(6, 1, make([]float64, 6))
|
||||
// Set is rows then columns
|
||||
a.Set(0, 0, 1)
|
||||
a.Set(2, 0, 1)
|
||||
|
||||
// ROBOTS ACTIVATE
|
||||
n.Activate(a, 2)
|
||||
Convey("The feed-forward results should be right...", func() {
|
||||
So(a.At(5, 0), ShouldAlmostEqual, 0.474, 0.01)
|
||||
So(a.At(4, 0), ShouldAlmostEqual, 0.525, 0.01)
|
||||
So(a.At(3, 0), ShouldAlmostEqual, 0.332, 0.01)
|
||||
|
||||
// Set the observed error on the output node
|
||||
e := mat64.NewDense(6, 1, make([]float64, 6))
|
||||
e.Set(5, 0, 1.0-a.At(5, 0))
|
||||
|
||||
// Run back-propagated error
|
||||
b := n.Error(a, e, 2)
|
||||
Convey("The back-prop results should be right...", func() {
|
||||
So(b.At(5, 0), ShouldAlmostEqual, 0.1311, 0.001)
|
||||
So(b.At(4, 0), ShouldAlmostEqual, -0.0065, 0.001)
|
||||
So(b.At(3, 0), ShouldAlmostEqual, -0.0087, 0.001)
|
||||
So(b.At(2, 0), ShouldAlmostEqual, 0.000)
|
||||
So(b.At(1, 0), ShouldAlmostEqual, 0.000)
|
||||
So(b.At(0, 0), ShouldAlmostEqual, 0.000)
|
||||
|
||||
Convey("The weight update results should be right...", func() {
|
||||
n.UpdateWeights(a, b, 0.9)
|
||||
for i := 1; i <= 6; i++ {
|
||||
for j := 1; j <= 6; j++ {
|
||||
v := n.GetWeight(i, j)
|
||||
fmt.Println(i, j, v)
|
||||
switch i {
|
||||
case 1:
|
||||
switch j {
|
||||
case 1:
|
||||
So(v, ShouldAlmostEqual, 1.000)
|
||||
case 4:
|
||||
So(v, ShouldAlmostEqual, 0.192, 0.001)
|
||||
case 5:
|
||||
So(v, ShouldAlmostEqual, -0.306, 0.001)
|
||||
default:
|
||||
So(v, ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
case 2:
|
||||
switch j {
|
||||
case 2:
|
||||
So(v, ShouldAlmostEqual, 1.000)
|
||||
case 4:
|
||||
So(v, ShouldAlmostEqual, 0.400, 0.001)
|
||||
case 5:
|
||||
So(v, ShouldAlmostEqual, 0.100, 0.001)
|
||||
default:
|
||||
So(v, ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
case 3:
|
||||
switch j {
|
||||
case 3:
|
||||
So(v, ShouldAlmostEqual, 1.000)
|
||||
case 4:
|
||||
So(v, ShouldAlmostEqual, -0.508, 0.001)
|
||||
case 5:
|
||||
So(v, ShouldAlmostEqual, 0.194, 0.001)
|
||||
default:
|
||||
So(v, ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
case 4:
|
||||
switch j {
|
||||
case 6:
|
||||
So(v, ShouldAlmostEqual, -0.261, 0.001)
|
||||
default:
|
||||
So(v, ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
case 5:
|
||||
switch j {
|
||||
case 6:
|
||||
So(v, ShouldAlmostEqual, -0.138, 0.001)
|
||||
default:
|
||||
So(v, ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
|
||||
default:
|
||||
So(v, ShouldAlmostEqual, 0.000)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
Convey("The bias update results should be right...", func() {
|
||||
n.UpdateBias(b, 0.9)
|
||||
So(n.GetBias(6), ShouldAlmostEqual, 0.218, 0.001)
|
||||
So(n.GetBias(5), ShouldAlmostEqual, 0.194, 0.001)
|
||||
So(n.GetBias(4), ShouldAlmostEqual, -0.408, 0.001)
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
19
neural/neural.go
Normal file
19
neural/neural.go
Normal file
@ -0,0 +1,19 @@
|
||||
//Package neural contains Neural Network functions.
|
||||
package neural
|
||||
|
||||
import (
|
||||
"github.com/gonum/matrix/mat64"
|
||||
)
|
||||
|
||||
type ActivationFunction func(float64) float64
|
||||
|
||||
// First function is always the forward activation function
|
||||
// Second function is always the backward activation function
|
||||
type NeuralFunction struct {
|
||||
Forward ActivationFunction
|
||||
Backward ActivationFunction
|
||||
}
|
||||
|
||||
// LayerFuncs are vectorised layer value transformation functions
|
||||
// (e.g. sigmoid). They must operate in-place.
|
||||
type LayerFunc func(*mat64.Dense)
|
||||
4
neural/xor.csv
Normal file
4
neural/xor.csv
Normal file
@ -0,0 +1,4 @@
|
||||
0,0,0
|
||||
0,1,1
|
||||
1,0,1
|
||||
1,1,0
|
||||
|
@ -23,4 +23,4 @@
|
||||
|
||||
*/
|
||||
|
||||
package trees
|
||||
package trees
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user