Neural Network from Scratch in Cangjie: Part 4 - 仓颉从头开始的神经网络：第四部分
Today, we will try to recreate the output layer for a typical classification network with the help of the `Softmax` activation function. It will let us normalize the output of the final layer in our network and display probabilities of which of the 3 classes the network thinks the input belongs to. For example, [0.25, 0.4, 0.35]. In this example, the network would be 40% sure that the second class is the correct class. These probabilities add up to 1.
Let's do another forward pass through our network, but this time add an output layer with 3 neurons that represent the 3 predicted classes in our simulated spiral data.
import matrix4cj.*
import std.collection.*
import std.random.*
import csv4cj.*
import std.os.posix.*
import std.fs.*
import std.convert.*
let random = Random(0) // seed = 0
main() {
let X: Array<Array<Float64>>
let y: Array<Int64>
(X, y) = getData()
let dense1 = Layer_Dense(2, 4, X.size)
let activation1 = Activation_ReLU()
let dense2 = Layer_Dense(4, 4, X.size)
let activation2 = Activation_ReLU()
let dense3 = Layer_Dense(4, 3, X.size)
dense1.forward(X)
activation1.forward(dense1.output)
println(activation1.output[..5])
dense2.forward(activation1.output)
activation2.forward(dense2.output)
println(activation2.output[..5])
dense3.forward(activation2.output)
println(dense3.output[..5])
}
class Activation_ReLU {
var output: Array<Array<Float64>>
public init() {
this.output = []
}
public func forward(inputs: Array<Array<Float64>>) {
let output = ArrayList<Array<Float64>>([])
for (array in inputs) {
output.append(maximum(array))
}
this.output = output.toArray()
}
private func maximum(input: Array<Float64>): Array<Float64> {
func clip(i: Float64): Float64 {
if (i > 0.0) {
return i
} else {
return 0.0
}
}
let output = input |> map {i => clip(i)} |> collectArray
return output
}
}
class Layer_Dense {
var weights: Matrix
var biases: Matrix
var output: Array<Array<Float64>>
public init(nInputs: Int64, nNeurons: Int64, batchSize: Int64) {
this.weights = Matrix(
Array<Array<Float64>>(nNeurons, {_ => Array<Float64>(nInputs, {_ => random.nextFloat64() * 0.01})}))
this.biases = Matrix(Array<Array<Float64>>(batchSize, {_ => Array<Float64>(nNeurons, {_ => 0.0})}))
this.output = []
}
public func forward(inputs: Array<Array<Float64>>) {
this.output = Matrix(inputs).times(this.weights.transpose()).plus(this.biases).getArray()
}
}
func getData() {
let yIdx: Int64 = 2
let X = ArrayList<Array<Float64>>([])
let y = ArrayList<Int64>([])
let path: String = getcwd()
let fileStream = File("${path}/test.csv", OpenOption.Open(true, false))
//打开文件流
if (fileStream.canRead()) {
//创建字符读取的解析流
let stream = UTF8ReaderStream(fileStream)
let reader = CSVReader(stream)
//创建格式化的解析参数
let format: CSVParseFormat = CSVParseFormat.DEFAULT
//创建解析器
let csvParser = CSVParser(reader, format)
for (csvRecord in csvParser) {
let values = csvRecord.getValues()
X.append(Array<Float64>(values[..yIdx].size, {j => Float64.parse(values[..yIdx][j].toString())}))
y.append(Int64.parse(values[yIdx].toString()))
}
fileStream.close()
}
return (X.toArray(), y.toArray())
}
Here, we add another dense layer `let dense3 = Layer_Dense(4, 3, X.size)` and do a forward pass with the output from the second layer's ReLU activation function. We get values for 3 classes to match the size of our `y`. Because we do batching, the size of the outputs = 300.
Our (2, 4, 4, 3) Network
At this point, the output doesn't make sense, as this is raw hidden layer output. In order to calculate the probabilities and display sensible results, we need the `Softmax` activation function.
First we need to do some preparations. The values coming from the last hidden layer need to be positive - we need to turn every possible negative value into positive, as probabilities cannot be negative. For that, we exponentiate the values. Although it scales the numbers, exponentiation doesn't affect the impact of of individual outputs, as higher inputs are translated into higher outputs, lower inputs are translated into lower outputs. The only effect is the elimination of negative numbers.
import std.math.*
func exponentiate(input: Array<Float64>): Array<Float64> {
let output = input |> map {i => exp(i)} |> collectArray
return output
}
main() {
let input = [4.8, 1.21, 2.385]
println(exponentiate(input))
}
>>> [121.510418, 3.353485, 10.859063]We can use the convenient `exp()` function from the `std.math` module.
Next, we need to normalize our values, so that they add up to 1. For that, we need to sum our exponentiated values and divide each value by the sum called `normBase`. We use the `reduce` array function to help us sum each element of the array.
Imagine we have this array: [1, 1, 1]. The `reduce` function would go from left to right, take the first 2 elements (x and y) and sum them up -> [2, 1]. Then the function does another pass and sums up the remaining 2 elements -> 3. Here is another example: [1, 2, 3, 4] -> [3, 3, 4] -> [6, 4] -> 10.
There is one last step that we need to do. Because `reduce` returns an Enum, in order to get the actual value of 10, we need to use the `getOrThrow()` method.
func exponentiateAndNormalize(input: Array<Float64>): Array<Float64> {
func sum(x: Float64, y: Float64): Float64 {
return x + y
}
let exponentiatedInput = input |> map {i => exp(i)} |> collectArray
let normBase = exponentiatedInput |> reduce(sum)
// 标准化
let probabilities = exponentiatedInput |> map {i => i / normBase.getOrThrow()} |> collectArray
return probabilities
}
main() {
let input = [4.8, 1.21, 2.385]
println(exponentiateAndNormalize(input))
}
>>> [0.895283, 0.024708, 0.080009]The output adds up to 1. This works as expected, but what if we have an array of arrays like with batched input?
func exponentiateAndNormalize(inputs: Array<Array<Float64>>): Array<Array<Float64>> {
func sum(x: Float64, y: Float64): Float64 {
return x + y
}
let outputs = ArrayList<Array<Float64>>([])
for (input in inputs) {
let exponentiatedInput = input |> map {i => exp(i)} |> collectArray
let normBase = exponentiatedInput |> reduce(sum)
// 标准化
let probabilities = exponentiatedInput |> map {i => i / normBase.getOrThrow()} |> collectArray
outputs.append(probabilities)
}
return outputs.toArray()
}
main() {
let inputs = [[4.8, 1.21, 2.385], [8.9, -1.81, 0.2], [1.41, 1.051, 0.026]]
println(exponentiateAndNormalize(inputs))
}
>>> [[0.895283, 0.024708, 0.080009], [0.999811, 0.000022, 0.000167], [0.513097, 0.358334, 0.128569]]Now, it is time to turn this into an activation class.
class Activation_Softmax {
var output: Array<Array<Float64>>
public init() {
this.output = []
}
public func forward(inputs: Array<Array<Float64>>) {
let output = ArrayList<Array<Float64>>([])
for (input in inputs) {
let exponentiatedInput = input |> map {i => exp(i)} |> collectArray
let normBase = exponentiatedInput |> reduce(sum)
// 标准化
let probabilities = exponentiatedInput |> map {i => i / normBase.getOrThrow()} |> collectArray
output.append(probabilities)
}
this.output = output.toArray()
}
private func sum(x: Float64, y: Float64): Float64 {
return x + y
}
}However, there is another problem that we need to address: "exploding" values. If we exponentiate the number 1000, we will overflow the stack, because the resulting number is just too big. How do we ensure that we won't get any exceptions in our network? We can subtract the highest number in the input array from every element in the very array. This won't change the output thanks to normalization but will help with the overflow problem. Let's change the part where we loop through the inputs array. First, we calculate the max value in a batch array, then we subtract it from every element in the same array. Finally, we use the subtracted array for exponentiation.
for (input in inputs) {
let maxValue = max(input)
let subtractedInput = input |> map {i: Float64 => i - maxValue.getOrThrow()} |> collectArray
let exponentiatedInput = subtractedInput |> map {i => exp(i)} |> collectArray
Here is the full code. We can now do a full pass through the network.
import matrix4cj.*
import std.collection.*
import std.random.*
import csv4cj.*
import std.os.posix.*
import std.fs.*
import std.convert.*
import std.math.*
let random = Random(0) // seed = 0
main() {
let X: Array<Array<Float64>>
let y: Array<Int64>
(X, y) = getData()
let dense1 = Layer_Dense(2, 4, X.size)
let activation1 = Activation_ReLU()
let dense2 = Layer_Dense(4, 4, X.size)
let activation2 = Activation_ReLU()
let dense3 = Layer_Dense(4, 3, X.size)
let activation3 = Activation_Softmax()
dense1.forward(X)
activation1.forward(dense1.output)
println(activation1.output[..5])
dense2.forward(activation1.output)
activation2.forward(dense2.output)
println(activation2.output[..5])
dense3.forward(activation2.output)
activation3.forward(dense3.output)
println(activation3.output[..5])
}
class Activation_Softmax {
var output: Array<Array<Float64>>
public init() {
this.output = []
}
public func forward(inputs: Array<Array<Float64>>) {
let output = ArrayList<Array<Float64>>([])
for (input in inputs) {
let maxValue = max(input)
let subtractedInput = input |> map {i: Float64 => i - maxValue.getOrThrow()} |> collectArray
let exponentiatedInput = subtractedInput |> map {i => exp(i)} |> collectArray
let normBase = exponentiatedInput |> reduce(sum)
// 标准化
let probabilities = exponentiatedInput |> map {i => i / normBase.getOrThrow()} |> collectArray
output.append(probabilities)
}
this.output = output.toArray()
}
private func sum(x: Float64, y: Float64): Float64 {
return x + y
}
}
class Activation_ReLU {
var output: Array<Array<Float64>>
public init() {
this.output = []
}
public func forward(inputs: Array<Array<Float64>>) {
let output = ArrayList<Array<Float64>>([])
for (array in inputs) {
output.append(maximum(array))
}
this.output = output.toArray()
}
private func maximum(input: Array<Float64>): Array<Float64> {
func clip(i: Float64): Float64 {
if (i > 0.0) {
return i
} else {
return 0.0
}
}
let output = input |> map {i => clip(i)} |> collectArray
return output
}
}
class Layer_Dense {
var weights: Matrix
var biases: Matrix
var output: Array<Array<Float64>>
public init(nInputs: Int64, nNeurons: Int64, batchSize: Int64) {
this.weights = Matrix(
Array<Array<Float64>>(nNeurons, {_ => Array<Float64>(nInputs, {_ => random.nextFloat64() * 0.01})}))
this.biases = Matrix(Array<Array<Float64>>(batchSize, {_ => Array<Float64>(nNeurons, {_ => 0.0})}))
this.output = []
}
public func forward(inputs: Array<Array<Float64>>) {
this.output = Matrix(inputs).times(this.weights.transpose()).plus(this.biases).getArray()
}
}
func getData() {
let yIdx: Int64 = 2
let X = ArrayList<Array<Float64>>([])
let y = ArrayList<Int64>([])
let path: String = getcwd()
let fileStream = File("${path}/test.csv", OpenOption.Open(true, false))
//打开文件流
if (fileStream.canRead()) {
//创建字符读取的解析流
let stream = UTF8ReaderStream(fileStream)
let reader = CSVReader(stream)
//创建格式化的解析参数
let format: CSVParseFormat = CSVParseFormat.DEFAULT
//创建解析器
let csvParser = CSVParser(reader, format)
for (csvRecord in csvParser) {
let values = csvRecord.getValues()
X.append(Array<Float64>(values[..yIdx].size, {j => Float64.parse(values[..yIdx][j].toString())}))
y.append(Int64.parse(values[yIdx].toString()))
}
fileStream.close()
}
return (X.toArray(), y.toArray())
}We can see that, because the network is still untrained, it outputs equal probabilities for the 3 classes to be true. These are also the network's confidence scores. This is because our weights are initialized randomly and biases are set to 0.
In the next tutorial, we will add the ability to calculate loss, or how wrong the network is with its predictions. This will give us the ability to adjust weights and biases in a way that decreases error and brings us a step closer to actually training it.
讨论HarmonyOS开发技术，专注于API与组件、DevEco Studio、测试、元服务和应用上架分发等。
更多推荐
10
0- 0
已为社区贡献7条内容
所有评论(0)