有偏置节点（With Bias Nodes）

import numpy as np

# 激活函数：Sigmoid
@np.vectorize
def sigmoid(x):
    return 1 / (1 + np.e ** -x)

activation_function = sigmoid

# 截断正态分布，用于初始化权重
from scipy.stats import truncnorm

def truncated_normal(mean=0, sd=1, low=0, upp=10):
    return truncnorm((low - mean) / sd,
                     (upp - mean) / sd,
                     loc=mean,
                     scale=sd)

# --- 神经网络类定义 ---
class NeuralNetwork:
    def __init__(self,
                 no_of_in_nodes,    # 输入层节点数
                 no_of_out_nodes,   # 输出层节点数
                 no_of_hidden_nodes,# 隐藏层节点数
                 learning_rate,     # 学习率
                 bias=None          # 偏置项，如果为None则不使用偏置
                ):
        self.no_of_in_nodes = no_of_in_nodes
        self.no_of_out_nodes = no_of_out_nodes
        self.no_of_hidden_nodes = no_of_hidden_nodes
        self.learning_rate = learning_rate
        self.bias = bias # 偏置值，例如 0.5 或 1.0

        self.create_weight_matrices()

    def create_weight_matrices(self):
        """
        一个初始化神经网络权重矩阵的方法，支持可选的偏置节点。
        """
        bias_node = 1 if self.bias else 0 # 如果有偏置，则偏置节点数为1，否则为0

        # 输入层到隐藏层的权重 (wih) 初始化
        # 权重初始化范围取决于输入节点数（包括偏置节点）
        rad = 1 / np.sqrt(self.no_of_in_nodes + bias_node)
        X = truncated_normal(mean=0,
                             sd=1,
                             low=-rad,
                             upp=rad)
        self.wih = X.rvs((self.no_of_hidden_nodes,
                          self.no_of_in_nodes + bias_node)) # 隐藏层节点数 x (输入节点数 + 偏置节点数)

        # 隐藏层到输出层的权重 (who) 初始化
        # 权重初始化范围取决于隐藏层节点数（包括偏置节点）
        rad = 1 / np.sqrt(self.no_of_hidden_nodes + bias_node)
        X = truncated_normal(mean=0, sd=1, low=-rad, upp=rad)
        self.who = X.rvs((self.no_of_out_nodes,
                          self.no_of_hidden_nodes + bias_node)) # 输出层节点数 x (隐藏层节点数 + 偏置节点数)

    def train(self, input_vector, target_vector):
        """
        训练方法：执行一次前向传播和一次反向传播。
        input_vector 和 target_vector 可以是元组、列表或 ndarray。
        """
        # 如果使用偏置，将偏置节点添加到输入向量的末尾
        if self.bias:
            input_vector = np.concatenate((input_vector, [self.bias]))

        # 将输入和目标向量转换为列向量
        input_vector = np.array(input_vector, ndmin=2).T
        target_vector = np.array(target_vector, ndmin=2).T

        # 前向传播：从输入层到隐藏层
        output_vector1 = np.dot(self.wih, input_vector)
        output_hidden = activation_function(output_vector1)

        # 如果使用偏置，将偏置节点添加到隐藏层输出的末尾
        if self.bias:
            output_hidden = np.concatenate((output_hidden, [[self.bias]]))

        # 前向传播：从隐藏层到输出层
        output_vector2 = np.dot(self.who, output_hidden)
        output_network = activation_function(output_vector2)

        # 计算输出误差
        output_errors = target_vector - output_network

        # 更新隐藏层到输出层的权重 (who)
        tmp = output_errors * output_network * (1.0 - output_network) # 输出层的梯度
        tmp = self.learning_rate * np.dot(tmp, output_hidden.T)
        self.who += tmp

        # 计算隐藏层误差（反向传播到隐藏层）
        hidden_errors = np.dot(self.who.T, output_errors)

        # 更新输入层到隐藏层的权重 (wih)
        tmp = hidden_errors * output_hidden * (1.0 - output_hidden) # 隐藏层的梯度

        # 如果有偏置，去除偏置节点对应的梯度，因为偏置节点没有输入误差反向传播
        if self.bias:
            x = np.dot(tmp, input_vector.T)[:-1, :] # 去除最后一列（偏置项）
        else:
            x = np.dot(tmp, input_vector.T)
        self.wih += self.learning_rate * x

    def run(self, input_vector):
        """
        运行方法：对给定输入执行前向传播以获得输出。
        input_vector 可以是元组、列表或 ndarray。
        """
        # 如果使用偏置，将偏置节点添加到输入向量的末尾
        if self.bias:
            input_vector = np.concatenate((input_vector, [self.bias])) # 注意这里偏置值用 self.bias

        input_vector = np.array(input_vector, ndmin=2).T

        # 前向传播：输入层到隐藏层
        output_vector = np.dot(self.wih, input_vector)
        output_vector = activation_function(output_vector)

        # 如果使用偏置，将偏置节点添加到隐藏层输出的末尾
        if self.bias:
            output_vector = np.concatenate((output_vector, [[self.bias]])) # 注意这里偏置值用 self.bias

        # 前向传播：隐藏层到输出层
        output_vector = np.dot(self.who, output_vector)
        output_vector = activation_function(output_vector)
        return output_vector

    def evaluate(self, data, labels):
        """
        评估网络在给定数据集上的表现。
        """
        corrects, wrongs = 0, 0
        for i in range(len(data)):
            res = self.run(data[i])
            res_max = res.argmax() # 预测结果的索引（即预测的数字）
            if res_max == int(labels[i][0]): # 将真实标签转换为整数进行比较
                corrects += 1
            else:
                wrongs += 1
        return corrects, wrongs

# --- 训练和测试（无偏置）---
# 导入之前保存的数据 (假设已经运行了前一部分代码并保存了数据)
import pickle
data_path = "data/mnist/"
try:
    with open(data_path + "pickled_mnist.pkl", "br") as fh:
        data = pickle.load(fh)
    train_imgs = data[0]
    test_imgs = data[1]
    train_labels = data[2]
    test_labels = data[3]
    train_labels_one_hot = data[4]
    test_labels_one_hot = data[5]
    image_size = 28
    image_pixels = image_size * image_size
    no_of_different_labels = 10
except FileNotFoundError:
    print("MNIST 数据文件未找到。请先运行前面部分的代码以生成 'pickled_mnist.pkl'。")
    exit()

print("--- 无偏置项的神经网络训练 ---")
ANN = NeuralNetwork(no_of_in_nodes=image_pixels,
                    no_of_out_nodes=10,
                    no_of_hidden_nodes=200, # 隐藏层节点数增加到200
                    learning_rate=0.1,
                    bias=None) # 不使用偏置

# 单次训练循环（遍历所有训练样本一次）
for i in range(len(train_imgs)):
    ANN.train(train_imgs[i], train_labels_one_hot[i])

print("测试集前20个样本的预测结果:")
for i in range(20):
    res = ANN.run(test_imgs[i])
    print(f"真实标签: {int(test_labels[i][0])}, 预测标签: {np.argmax(res)}, 最大预测概率: {np.max(res):.4f}")

corrects_train, wrongs_train = ANN.evaluate(train_imgs, train_labels)
print("训练准确率: ", corrects_train / (corrects_train + wrongs_train))
corrects_test, wrongs_test = ANN.evaluate(test_imgs, test_labels)
print("测试准确率: ", corrects_test / (corrects_test + wrongs_test))

print("\n--- 带偏置项和 Epochs 的神经网络训练 ---")

# --- 带有偏置项和 Epochs 的版本 ---
class NeuralNetwork: # 重新定义类，因为之前的示例只是更改了train方法，这里为了完整性重新包含整个类
    def __init__(self,
                 no_of_in_nodes,
                 no_of_out_nodes,
                 no_of_hidden_nodes,
                 learning_rate,
                 bias=None
                ):
        self.no_of_in_nodes = no_of_in_nodes
        self.no_of_out_nodes = no_of_out_nodes
        self.no_of_hidden_nodes = no_of_hidden_nodes
        self.learning_rate = learning_rate
        self.bias = bias
        self.create_weight_matrices()

    def create_weight_matrices(self):
        bias_node = 1 if self.bias else 0
        rad = 1 / np.sqrt(self.no_of_in_nodes + bias_node)
        X = truncated_normal(mean=0, sd=1, low=-rad, upp=rad)
        self.wih = X.rvs((self.no_of_hidden_nodes, self.no_of_in_nodes + bias_node))

        rad = 1 / np.sqrt(self.no_of_hidden_nodes + bias_node)
        X = truncated_normal(mean=0, sd=1, low=-rad, upp=rad)
        self.who = X.rvs((self.no_of_out_nodes, self.no_of_hidden_nodes + bias_node))

    def train_single(self, input_vector, target_vector):
        """
        单样本训练，带有偏置项处理。
        """
        # 如果使用偏置，将偏置节点添加到输入向量的末尾
        if self.bias:
            input_vector = np.concatenate((input_vector, [self.bias]))

        input_vector = np.array(input_vector, ndmin=2).T
        target_vector = np.array(target_vector, ndmin=2).T

        output_vector1 = np.dot(self.wih, input_vector)
        output_hidden = activation_function(output_vector1)

        if self.bias:
            output_hidden = np.concatenate((output_hidden, [[self.bias]]))

        output_vector2 = np.dot(self.who, output_hidden)
        output_network = activation_function(output_vector2)

        output_errors = target_vector - output_network

        tmp = output_errors * output_network * (1.0 - output_network)
        tmp = self.learning_rate * np.dot(tmp, output_hidden.T)
        self.who += tmp

        hidden_errors = np.dot(self.who.T, output_errors)

        tmp = hidden_errors * output_hidden * (1.0 - output_hidden)
        if self.bias:
            x = np.dot(tmp, input_vector.T)[:-1, :]
        else:
            x = np.dot(tmp, input_vector.T)
        self.wih += self.learning_rate * x

    def train(self, data_array, labels_one_hot_array, epochs=1, intermediate_results=False):
        """
        多 epoch 训练，可以保存中间权重。
        """
        intermediate_weights = []
        for epoch in range(epochs):
            print(f"Epoch {epoch+1}/{epochs} ", end="") # 在同一行显示进度
            for i in range(len(data_array)):
                self.train_single(data_array[i], labels_one_hot_array[i])
            
            # 在每个 epoch 结束时，评估并打印准确率
            corrects, wrongs = self.evaluate(train_imgs, train_labels)
            train_accuracy = corrects / (corrects + wrongs)
            corrects, wrongs = self.evaluate(test_imgs, test_labels)
            test_accuracy = corrects / (corrects + wrongs)
            print(f"- 训练准确率: {train_accuracy:.4f}, 测试准确率: {test_accuracy:.4f}")

            if intermediate_results:
                intermediate_weights.append((self.wih.copy(), self.who.copy()))
        return intermediate_weights

    def run(self, input_vector):
        """
        运行方法，带有偏置项处理。
        """
        if self.bias:
            input_vector = np.concatenate((input_vector, [self.bias]))
        input_vector = np.array(input_vector, ndmin=2).T

        output_vector = np.dot(self.wih, input_vector)
        output_vector = activation_function(output_vector)

        if self.bias:
            output_vector = np.concatenate((output_vector, [[self.bias]]))

        output_vector = np.dot(self.who, output_vector)
        output_vector = activation_function(output_vector)
        return output_vector

    def evaluate(self, data, labels):
        """
        评估网络在给定数据集上的表现。
        """
        corrects, wrongs = 0, 0
        for i in range(len(data)):
            res = self.run(data[i])
            res_max = res.argmax()
            if res_max == int(labels[i][0]): # 确保标签是整数
                corrects += 1
            else:
                wrongs += 1
        return corrects, wrongs

# --- 带有偏置项的训练示例 ---
epochs_with_bias = 12
network = NeuralNetwork(no_of_in_nodes=image_pixels,
                        no_of_out_nodes=10,
                        no_of_hidden_nodes=100, # 隐藏层节点数，这里使用100
                        learning_rate=0.1,
                        bias=0.5) # 使用偏置项，值为 0.5

print(f"\n使用 {epochs_with_bias} 个 epoch 训练神经网络 (包含偏置项):")
weights = network.train(train_imgs,
                        train_labels_one_hot,
                        epochs=epochs_with_bias,
                        intermediate_results=True)

# 打印每个 epoch 的准确率
print("\n每个 epoch 的训练和测试准确率:")
for epoch in range(epochs_with_bias):
    print(f"epoch: {epoch}")
    # 恢复该 epoch 结束时的权重
    network.wih = weights[epoch][0]
    network.who = weights[epoch][1]
    
    corrects_train, wrongs_train = network.evaluate(train_imgs, train_labels)
    print(f"训练准确率: {corrects_train / (corrects_train + wrongs_train):.4f}")
    
    corrects_test, wrongs_test = network.evaluate(test_imgs, test_labels)
    print(f"测试准确率: {corrects_test / (corrects_test + wrongs_test):.4f}")

print("\n--- 大规模参数搜索和结果保存（到nist_tests.csv）---")
print("注意：此部分代码运行时间较长。")

# 循环遍历不同参数组合进行训练和评估
# 确保 'nist_tests.csv' 文件可以写入
with open("nist_tests.csv", "w") as fh_out:
    for hidden_nodes in [20, 50, 100, 120, 150]:
        for learning_rate in [0.01, 0.05, 0.1, 0.2]:
            for bias_val in [None, 0.5]: # 注意这里我把变量名从 bias 改为 bias_val 以避免与 NeuralNetwork.bias 混淆
                print(f"测试: hidden_nodes={hidden_nodes}, learning_rate={learning_rate}, bias={bias_val}")
                
                current_network = NeuralNetwork(no_of_in_nodes=image_pixels,
                                                no_of_out_nodes=10,
                                                no_of_hidden_nodes=hidden_nodes,
                                                learning_rate=learning_rate,
                                                bias=bias_val)
                
                current_weights = current_network.train(train_imgs,
                                                        train_labels_one_hot,
                                                        epochs=epochs_with_bias, # 使用与前面相同的 epoch 数量
                                                        intermediate_results=True)
                
                # 遍历每个 epoch 的结果并写入文件
                for epoch_idx in range(epochs_with_bias):
                    print("*", end="") # 打印星号表示进度
                    current_network.wih = current_weights[epoch_idx][0]
                    current_network.who = current_weights[epoch_idx][1]
                    
                    train_corrects, train_wrongs = current_network.evaluate(train_imgs, train_labels)
                    test_corrects, test_wrongs = current_network.evaluate(test_imgs, test_labels)
                    
                    # 格式化输出字符串
                    outstr = f"{hidden_nodes} {learning_rate} {bias_val} {epoch_idx} "
                    outstr += f"{train_corrects / (train_corrects + train_wrongs):.6f} " # 训练准确率
                    outstr += f"{train_wrongs / (train_corrects + train_wrongs):.6f} "  # 训练错误率
                    outstr += f"{test_corrects / (test_corrects + test_wrongs):.6f} "   # 测试准确率
                    outstr += f"{test_wrongs / (test_corrects + test_wrongs):.6f}"      # 测试错误率
                    
                    fh_out.write(outstr + "\n")
                    fh_out.flush() # 立即写入文件，防止数据丢失
                print("") # 换行，以便下一个参数组合的输出
print("\n所有测试结果已写入 nist_tests.csv 文件。")

--- 无偏置项的神经网络训练 ---
测试集前20个样本的预测结果:
真实标签: 7, 预测标签: 7, 最大预测概率: 0.9951
...
训练准确率:  0.9556
测试准确率:  0.9544

--- 带偏置项和 Epochs 的神经网络训练 ---

使用 12 个 epoch 训练神经网络 (包含偏置项):
Epoch 1/12 - 训练准确率: 0.9428, 测试准确率: 0.9415
Epoch 2/12 - 训练准确率: 0.9597, 测试准确率: 0.9548
Epoch 3/12 - 训练准确率: 0.9673, 测试准确率: 0.9599
Epoch 4/12 - 训练准确率: 0.9693, 测试准确率: 0.9601
Epoch 5/12 - 训练准确率: 0.9720, 测试准确率: 0.9631
Epoch 6/12 - 训练准确率: 0.9751, 测试准确率: 0.9659
Epoch 7/12 - 训练准确率: 0.9770, 测试准确率: 0.9662
Epoch 8/12 - 训练准确率: 0.9768, 测试准确率: 0.9644
Epoch 9/12 - 训练准确率: 0.9766, 测试准确率: 0.9643
Epoch 10/12 - 训练准确率: 0.9771, 测试准确率: 0.9643
Epoch 11/12 - 训练准确率: 0.9780, 测试准确率: 0.9627
Epoch 12/12 - 训练准确率: 0.9787, 测试准确率: 0.9638

每个 epoch 的训练和测试准确率:
epoch: 0
训练准确率: 0.9428
测试准确率: 0.9415
... (后续 epoch 的准确率)