This experiment aims to explore the effectiveness of Convolutional Neural Networks (CNN) in captcha recognition. Considering captcha complexity, CNN is used to develop a model that recognizes various captchas (including distorted text, noisy backgrounds, etc.) and assess its practical application feasibility.

Generating the Dataset

For a preliminary trial, we opt for a simpler captcha generating library:

go get github.com/afocus/captcha

Checking the installed fonts:

# For Linux, use fc-list
dir C:\Windows\Fonts

Writing a function to generate the dataset:

# Linux可以使用fc-listdir C:\Windows\Fonts

package main
import (
"fmt""image/color""image/png""os""github.com/afocus/captcha"
)
funcmain() {
// 创建一个验证码对象cap:=captcha.New()
// 设置验证码的字体iferr:=cap.SetFont("MSYH.TTC"); err!=nil {
panic(err.Error())
}
// 设置验证码的大小cap.SetSize(128, 64)
// 设置验证码的干扰强度cap.SetDisturbance(captcha.MEDIUM)
// 设置验证码的前景色cap.SetFrontColor(color.RGBA{255, 255, 255, 255})
// 设置验证码的背景色cap.SetBkgColor(
color.RGBA{255, 0, 0, 255}, // 红色
color.RGBA{0, 255, 0, 255}, // 绿色
color.RGBA{0, 0, 255, 255}, // 蓝色
color.RGBA{255, 255, 0, 255}, // 黄色
color.RGBA{255, 0, 255, 255}, // 紫色
color.RGBA{0, 255, 255, 255}, // 青色
color.RGBA{255, 165, 0, 255}, // 橙色
color.RGBA{128, 128, 128, 255} // 灰色
)
dirName:="captcha"// 检查目录是否存在if_, err:=os.Stat(dirName); os.IsNotExist(err) {
// 如果目录不存在，则创建它errDir:=os.MkdirAll(dirName, 0755)
iferrDir!=nil {
fmt.Println()
panic(fmt.Sprintf("error creating directory: %v", errDir))
}
}
// 生成1000张验证码图片fori:=0; i<1000; i++ {
img, str:=cap.Create(4, captcha.ALL)
fileName:=fmt.Sprintf("./"+dirName+"/%d_%s.png", i, str)
file, err:=os.Create(fileName)
iferr!=nil {
panic(err.Error())
}
png.Encode(file, img)
file.Close()
}
}

Generated captchas:

Introducing Dependencies

importtorchimporttorch.nnasnnimporttorch.optimasoptimfromtorch.utils.dataimportDatasetfromPILimportImageimportosimporttorchvision.transformsastransformsimportmatplotlib.pyplotasplt

Data Preprocessing

Since the captchas aren't square, to preprocess the images into square shapes, we first perform padding and then scaling. Padding ensures the images have equal width and height without altering the content.

To achieve a 128x128 square image:

1. Use transforms.Resize to scale the shorter side of the image, maintaining its aspect ratio.
2. Employ transforms.Pad to make the longer side equal to the shorter one, achieving a square shape.
3. Finally, apply transforms.Resize again to adjust to the desired size.

Example code:

fromtorchvisionimporttransformsdefpad_to_square(img):
# 获取图像的尺寸w, h=img.size# 计算要填充的最大边max_dim=max(w, h)
# 计算左右和上下需要填充的尺寸pad_w= (max_dim-w) //2pad_h= (max_dim-h) //2# 返回填充后的图像returntransforms.Pad((pad_w, pad_h))(img)
transform=transforms.Compose([
transforms.Lambda(lambdax: x.convert("RGB")), # 转换为RGB格式transforms.Lambda(pad_to_square), # 将图像填充为正方形transforms.Resize((128, 128)), # 将图像调整为所需尺寸transforms.ToTensor(),
])

This method first pads the image into a square shape, then resizes it. This approach ensures the content of the image isn't distorted due to varying scaling ratios, maintaining consistency in the image content.

Defining the Dataset Class

As the dataset is self-generated, we need to write our dataset collection class using PyTorch's torch.utils.data.Dataset, implementing the __len__ and __getitem__ methods.

# 定义分类nums=list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
# 创建字符到整数的映射（字典推导）char_to_int= {char: ifori, charinenumerate(nums)}
classCaptchaDataset(Dataset):
def__init__(self, root_dir, transform=None):
self.root_dir=root_dirself.transform=transformself.image_files= [fforfinos.listdir(
root_dir) iff.endswith('.png')]
def__len__(self):
returnlen(self.image_files)
def__getitem__(self, idx):
img_name=os.path.join(self.root_dir, self.image_files[idx])
image=Image.open(img_name)
label_str=self.image_files[idx].split('_')[1].split('.')[
0] # 提取验证码的名称作为 Labellabel=torch.tensor([char_to_int[char]
forcharinlabel_str], dtype=torch.long) # 转换为整数张量（就是取得当前这个 char 在字符串的索引）ifself.transform:
image=self.transform(image)
# label: 是一个一维张量，维度为 [sequence_length]，其中 sequence_length 是字符序列的长度。每个元素是一个类别标签（索引）。returnimage, label# 训练数据集train_dataset=CaptchaDataset(root_dir='captcha', transform=transform)
trainloader=torch.utils.data.DataLoader(
train_dataset, batch_size=64, shuffle=True)
# 测试数据集test_dataset=CaptchaDataset(root_dir='test_captcha', transform=transform)
testloader=torch.utils.data.DataLoader(
test_dataset, batch_size=64, shuffle=True)

Defining Related Functions

Next, we define functions for training, evaluation, and plotting.

Defining the Loss Function and Optimizer

First, we define the loss function and optimizer.

# 定义损失函数和优化器criterion=nn.CrossEntropyLoss()
optimizer=optim.Adam(model.parameters(), lr=0.005)
# 定义StepLR调度器，每10个epoch，学习率乘以gamma=0.9scheduler=optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.9)

Plotting Convergence per Epoch

Defining a function to plot convergence for each epoch.

defupdate_plots(losses, accuracies):
"""实时更新并显示损失和准确率的图表"""# 清除先前的图表plt.clf()
fig, axs=plt.subplots(2)
# 绘制损失曲线axs[0].plot(losses, 'r', label='Training Loss')
axs[0].set_title('Training Loss')
axs[0].set_xlabel('Epochs')
axs[0].set_ylabel('Loss')
axs[0].legend()
# 绘制准确率曲线axs[1].plot(accuracies, 'g', label='Training Accuracy')
axs[1].set_title('Training Accuracy')
axs[1].set_xlabel('Epochs')
axs[1].set_ylabel('Accuracy (%)')
axs[1].legend()
# 使用plt.pause以实时更新图表plt.pause(0.001)
# 保持图表开放plt.show(block=False)

Defining the Training Function

Then, we define the training function, which is a standard training process.

deftrain_model(model, trainloader, criterion, optimizer, epochs, device):
train_losses= []
train_accuracies= [] # 用于记录每个周期的准确率forepochinrange(epochs):
running_loss=0.0correct=0# 记录这个周期中正确预测的字符数量total=0# 记录这个周期中总的字符数量for_, datainenumerate(trainloader, 0):
inputs, labels=data[0].to(device), data[1].to(device)
optimizer.zero_grad()
outputs=model(inputs)
# outputs 的维度是 [batch_size, sequence_length, num_chars]# labels 的维度是 [batch_size, sequence_length]# outputs[:, i, :]：此操作从 outputs 张量中选取第 i 个字符的预测，对于批次中的所有样本。# 其维度为 [batch_size, num_chars]，表示每个样本的第 i 个字符的类别预测分数。# labels[:, i]：此操作从 labels 张量中选取第 i 个字符的实际类别标签，对于批次中的所有样本。# 其维度为 [batch_size]，表示每个样本的第 i 个字符的实际类别。## 通过遍历 sequence_length，我们可以为每个字符位置计算损失。loss=sum([criterion(outputs[:, i, :], labels[:, i])
foriinrange(labels.size(1))])
loss.backward()
optimizer.step()
running_loss+=loss.item()
# 计算这个批次的准确率_, predicted=torch.max(outputs.data, 2)
total+=labels.size(0) *labels.size(1)
correct+= (predicted==labels).sum().item()
# 每个epoch结束后，调用scheduler的step方法来更新学习率scheduler.step()
# 打印当前学习率（可选）print(f"Epoch {epoch+1}, LR: {scheduler.get_last_lr()[0]}")
train_losses.append(running_loss/len(trainloader))
epoch_accuracy=100*correct/total# 计算这个周期的准确率train_accuracies.append(epoch_accuracy) # 将这个周期的准确率添加到列表中update_plots(train_losses, train_accuracies)	# 更新图表print(f"Epoch {epoch+1}, Loss: {running_loss/len(trainloader)}, Accuracy: {epoch_accuracy:.2f}%")
returntrain_losses

Defining the Evaluation Function

deftest_model(model, testloader, device):
correct=0total=0withtorch.no_grad():
fordataintestloader:
images, labels=data[0].to(device), data[1].to(device)
outputs=model(images)
# torch.max 函数用于返回指定维度上的最大值。# 在这里，我们希望得到每个验证码字符位置的最大分数对应的字符类别的索引。# 因为 outputs 的形状是 [batch_size, sequence_length, num_chars]，我们沿着第 2 个维度（索引为2）取最大值，这样我们可以为每个样本的每个字符位置得到一个最大分数的索引。_, predicted=torch.max(outputs.data, 2)
# 因为 labels 的维度是 [batch_size, sequence_length]# 这一行代码计算了总的字符数量。labels.size(0) 是批次中的样本数，而 labels.size(1) 是每个验证码的字符长度。# 所以，两者的乘积给出了这个批次中所有验证码字符的总数。total+=labels.size(0) *labels.size(1)
# predicted == labels 返回一个布尔值的张量，形状与 labels 相同。如果预测正确，对应位置的值为 True，否则为 False。# .sum() 计算这个批次中正确预测的字符数量。# .item() 方法将单元素张量的值转换为 Python 数值。correct+= (predicted==labels).sum().item()
return100*correct/total

Defining the Plotting Function

Defining a function to plot training loss and accuracy during the training process.

defplot_training(train_losses, test_accuracies):
fig, ax1=plt.subplots()
color='tab:red'ax1.set_xlabel('Epochs')
ax1.set_ylabel('Train Loss', color=color)
ax1.plot(train_losses, color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax2=ax1.twinx()
color='tab:blue'ax2.set_ylabel('Test Accuracy', color=color)
ax2.plot(test_accuracies, color=color)
ax2.tick_params(axis='y', labelcolor=color)
fig.tight_layout()
plt.show()

Training Process Flow

epochs=10# 训练模型train_losses=train_model(
model, trainloader, criterion, optimizer, epochs, device)
# 测试模型test_accuracies= [test_model(model, testloader, device)
for_inrange(epochs)]
# 绘制训练损失和测试精度plot_training(train_losses, test_accuracies)
print("Finished Training")
# 评估模型test_accuracy=test_model(model, testloader, device)
print(f"Accuracy on test set: {test_accuracy}%")
# 保存模型torch.save(model.state_dict(), 'captcha_model.pth')

Supplement: Using TensorBoard

To integrate TensorBoard with PyTorch, use the torch.utils.tensorboard package.

1. Import SummaryWriter for logging in TensorBoard.
2. Create a SummaryWriter object.
3. Record loss and accuracy in the training loop using writer.add_scalar.
4. Close the writer after training.

Here's the modified code section, with changes only related to TensorBoard.

fromtorch.utils.tensorboardimportSummaryWriterfromdatetimeimportdatetime# ... [其它代码保持不变]# 创建一个 SummaryWriter 对象，这里目录得不一样now=datetime.now()
logdir="runs/"+now.strftime("%Y%m%d-%H%M%S") +""writer=SummaryWriter(logdir)
# ... [其它代码保持不变]deftrain_model(model, trainloader, criterion, optimizer, epochs, device):
train_losses= []
train_accuracies= []
forepochinrange(epochs):
running_loss=0.0correct=0total=0for_, datainenumerate(trainloader, 0):
# ... [其它代码保持不变]scheduler.step()
epoch_loss=running_loss/len(trainloader)
epoch_accuracy=100*correct/total# 使用 SummaryWriter 记录损失和准确率writer.add_scalar('Training Loss', epoch_loss, epoch)
writer.add_scalar('Training Accuracy', epoch_accuracy, epoch)
train_losses.append(epoch_loss)
train_accuracies.append(epoch_accuracy)
# 打印日志，不再更新图表print(f"Epoch {epoch+1}, Loss: {epoch_loss}, Accuracy: {epoch_accuracy:.2f}%")
returntrain_losses# ... [其它代码保持不变]# 在所有训练和评估完成后关闭 writerwriter.close()

Now, use TensorBoard to view the graphs for training loss and accuracy.

To run TensorBoard:

tensorboard --logdir=runs

Access TensorBoard in the browser (usually at http://localhost:6006/) to view the graphs for training loss and accuracy.

Defining the Model

Strategies for debugging in deep learning:

Start with a simple model and gradually increase its complexity. Divide this process into five steps:

1. Start with the simplest model.
2. Successfully build the model and reproduce results.
3. Decompose bias terms and progressively fit the data.
4. Use coarse-to-fine random search for hyperparameter optimization.
5. Increase model size for underfitting or add data/adjust for overfitting.

First Version of the Model

First, we define a basic LeNet model.

classCaptchaModel(nn.Module):
def__init__(self, num_chars):
"""验证码识别模型 Args: num_chars (int): 字符类别的数量 """super(CaptchaModel, self).__init__()
# 假设验证码由4个字符组成self.num_chars=num_chars# 输入的是 3 维的图像，输出的是 32 维的特征图self.conv1=nn.Conv2d(3, 32, kernel_size=3, padding=1)
self.conv2=nn.Conv2d(32, 64, kernel_size=3, padding=1)
self.conv3=nn.Conv2d(64, 128, kernel_size=3, padding=1)
self.conv4=nn.Conv2d(128, 256, kernel_size=3, padding=1)
self.pool=nn.MaxPool2d(2, 2)
# 对于 128x128 输入，经过4次下采样后，尺寸将变为 8x8self.fc1=nn.Linear(256*8*8, 1024)
# 输出层，把 1024 维的特征转换为 4 * num_chars 维self.fc2=nn.Linear(1024, self.num_chars*4) # 4 是验证码包含的字符数defforward(self, x):
x=self.pool(F.relu(self.conv1(x)))
x=self.pool(F.relu(self.conv2(x)))
x=self.pool(F.relu(self.conv3(x)))
x=self.pool(F.relu(self.conv4(x)))
x=x.view(-1, 256*8*8)
# 全连接层x=F.relu(self.fc1(x))
x=self.fc2(x)
returnx.view(x.size(0), 4, self.num_chars)

After training for 10 epochs, we find the curve as shown below. The Loss continues to decrease, but the Accuracy fluctuates greatly.

Adjusting the Dataset

Initially, we used three channels of the dataset, but since both the background and font colors of our captchas

are random, we can convert them to grayscale. This reduces the model's complexity and speeds up training.

# 数据加载和预处理transform=transforms.Compose([
transforms.Lambda(lambdax: x.convert("RGB")), # 转换为RGB格式transforms.Grayscale(num_output_channels=1), # 转换为灰度图像transforms.Lambda(pad_to_square), # 将图像填充为正方形transforms.Resize((128, 128)), # 将图像调整为所需尺寸transforms.ToTensor(),
])

Simply modify the model to change the input channel to 1.

self.conv1=nn.Conv2d(1, 32, kernel_size=3, padding=1)

As seen in the smoother curve (red line) below, the Loss decreases slowly, and the Accuracy is not high.

The Loss stops decreasing beyond a point, indicating underfitting. This suggests the model is too simple for this complex classification task. Therefore, we switch to a more complex model.

Second Version of the Model

To accommodate the complex captcha, we consider:

1. Increasing convolutional depth.
2. Adding Batch Normalization and Dropout.
3. Using Adaptive Pooling.

Here's the modified model code:

classCaptchaModel(nn.Module):
def__init__(self, num_chars):
super(CaptchaModel, self).__init__()
self.num_chars=num_chars# 增加卷积层的深度self.conv1=nn.Conv2d(3, 32, kernel_size=3, padding=1)
self.bn1=nn.BatchNorm2d(32)
self.conv2=nn.Conv2d(32, 64, kernel_size=3, padding=1)
self.bn2=nn.BatchNorm2d(64)
self.conv3=nn.Conv2d(64, 128, kernel_size=3, padding=1)
self.bn3=nn.BatchNorm2d(128)
self.conv4=nn.Conv2d(128, 256, kernel_size=3, padding=1)
self.bn4=nn.BatchNorm2d(256)
self.conv5=nn.Conv2d(256, 512, kernel_size=3, padding=1)
self.bn5=nn.BatchNorm2d(512)
self.pool=nn.MaxPool2d(2, 2)
# 使用Adaptive Poolingself.adaptive_pool=nn.AdaptiveAvgPool2d((1, 1))
self.fc1=nn.Linear(512, 1024)
self.dropout=nn.Dropout(0.5)
self.fc2=nn.Linear(1024, self.num_chars*4)
defforward(self, x):
x=self.pool(F.relu(self.bn1(self.conv1(x))))
x=self.pool(F.relu(self.bn2(self.conv2(x))))
x=self.pool(F.relu(self.bn3(self.conv3(x))))
x=self.pool(F.relu(self.bn4(self.conv4(x))))
x=self.adaptive_pool(F.relu(self.bn5(self.conv5(x))))
x=x.view(-1, 512)
# 全连接层x=F.relu(self.fc1(x))
x=self.dropout(x) # 在全连接层后增加 Dropoutx=self.fc2(x)
returnx.view(x.size(0), 4, self.num_chars)

The Loss drops faster, and Accuracy improves significantly.

However, as model complexity increases, so does the training time.

Adjusting Batch Size

Initially set to 64, we experiment with larger Batch Sizes.

:::tip In fact, batch size is the easiest of all hyperparameters to adjust and should be determined first. It's usually chosen first before adjusting other hyperparameters.:::

First adjusting to 256:

train_dataset=CaptchaDataset(root_dir='captcha', transform=transform)
trainloader=torch.utils.data.DataLoader(
train_dataset, batch_size=256, shuffle=True)
test_dataset=CaptchaDataset(root_dir='test_captcha', transform=transform)
testloader=torch.utils.data.DataLoader(
test_dataset, batch_size=256, shuffle=True)

GPU usage increases significantly, indicating a higher computational load due to the larger Batch Size.

After adjusting to 256, we notice a decrease in training time but also a drop in accuracy. This is due to the larger Batch Size reducing the model's generalization ability.

Adjusting to 128 doesn't work either, so we revert to 64.

Adjusting Epochs

To improve accuracy, we increase the epochs from 10 to 50.

Beyond 30 epochs, the Loss decreases slowly, indicating model convergence. Thus, training is stopped.

The final result is a 50% accuracy rate, which is low. Therefore, we attempt to increase the training dataset size.

As model complexity increases, the Loss decreases rapidly, and Accuracy improves significantly.

Fourth Version of the Model

We now try a ResNet residual network for even greater complexity.

importtorch.nn.functionalasFimporttorch.nnasnn# 定义一个残差块classResidualBlock(nn.Module):
def__init__(self, in_channels, out_channels, stride=1):
super(ResidualBlock, self).__init__()
# 主要的卷积路径# 第一层卷积self.conv1=nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1)
self.bn1=nn.BatchNorm2d(out_channels)
# 第二层卷积self.conv2=nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)
self.bn2=nn.BatchNorm2d(out_channels)
# 快捷路径（shortcut）# 当输入输出的维度不一致时，我们需要调整其维度（可能是由于 stride 不为 1 或 in_channels != out_channels)self.shortcut=nn.Sequential()
ifstride!=1orin_channels!=out_channels:
self.shortcut=nn.Sequential(
# 1x1卷积用于调整维度nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride),
nn.BatchNorm2d(out_channels)
)
defforward(self, x):
out=F.relu(self.bn1(self.conv1(x)))
out=self.bn2(self.conv2(out))
out+=self.shortcut(x) # 添加快捷路径out=F.relu(out)
returnoutclassCaptchaModel(nn.Module):
def__init__(self, num_chars):
super(CaptchaModel, self).__init__()
self.num_chars=num_chars# 初始卷积层self.conv1=nn.Conv2d(1, 32, kernel_size=3, padding=1)
self.bn1=nn.BatchNorm2d(32)
# 最大池化层self.pool=nn.MaxPool2d(2, 2)
# 残差块self.res_block1=ResidualBlock(32, 64)
self.res_block2=ResidualBlock(64, 128)
self.res_block3=ResidualBlock(128, 256)
self.res_block4=ResidualBlock(256, 512)
# 自适应池化层self.adaptive_pool=nn.AdaptiveAvgPool2d((1, 1))
# 全连接层self.fc1=nn.Linear(512, 1024)
self.dropout=nn.Dropout(0.5) # 防止过拟合self.fc2=nn.Linear(1024, self.num_chars*4)
defforward(self, x):
x=self.pool(F.relu(self.bn1(self.conv1(x))))
x=self.pool(self.res_block1(x)) # 通过残差块x=self.pool(self.res_block2(x))
x=self.pool(self.res_block3(x))
x=self.adaptive_pool(self.res_block4(x))
x=x.view(-1, 512) # 展平x=F.relu(self.fc1(x))
x=self.dropout(x)
x=self.fc2(x)
returnx.view(x.size(0), 4, self.num_chars) # 重新调整形状以匹配预期的输出形状

After training with the new model, the curve shows significant improvement over previous models.

The trade-off is increased training time due to the model's complexity.

References

• Why is your model performing poorly, and what are the techniques for tuning deep learning?
• Solving and Identifying Underfitting in Deep Learning