BinaryTextClassification.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Tensorflow;
using Tensorflow.NumPy;
using static Tensorflow.Binding;
using static Tensorflow.KerasApi;

namespace TensorFlowNET.Examples;

/// <summary>
/// This tutorial demonstrates text classification starting from plain text files stored on disk.
/// You'll train a binary classifier to perform sentiment analysis on an IMDB dataset. 
/// At the end of the notebook, there is an exercise for you to try, in which you'll train a 
/// multiclass classifier to predict the tag for a programming question on Stack Overflow.
/// https://www.tensorflow.org/tutorials/keras/text_classification
/// </summary>
public class BinaryTextClassification : SciSharpExample, IExample
{
 NDArray train_data, train_labels, test_data, test_labels;

 public ExampleConfig InitConfig()
 => Config = new ExampleConfig
 {
 Name = "Binary Text Classification",
 Enabled = false
 };

 public bool Run()
 {
 PrepareData();

 // Console.WriteLine($"Training entries: {train_data.shape[0]}, labels: {train_labels.shape[0]}");

 // A dictionary mapping words to an integer index
 /*train_data = keras.preprocessing.sequence.pad_sequences(train_data,
 value: word_index["<PAD>"],
 padding: "post",
 maxlen: 256);

 test_data = keras.preprocessing.sequence.pad_sequences(test_data,
 value: word_index["<PAD>"],
 padding: "post",
 maxlen: 256);*/

 // input shape is the vocabulary count used for the movie reviews (10,000 words)

 var model = keras.Sequential();
 //var layer = tf.keras.layers.Embedding(vocab_size, 16);
 //model.add(layer);

 return false;
 }

 public override void PrepareData()
 {
 // tf.debugging.set_log_device_placement(true);
 string url = "https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz";
 var dataset = keras.utils.get_file("aclImdb_v1.tar.gz", url,
 untar: true,
 cache_dir: Path.GetTempPath(),
 cache_subdir: "aclImdb_v1");
 var data_dir = Path.Combine(dataset, "aclImdb");
 var train_dir = Path.Combine(data_dir, "train");

 int batch_size = 32;
 int seed = 42;
 var raw_train_ds = keras.preprocessing.text_dataset_from_directory(
 train_dir,
 batch_size: batch_size,
 validation_split: 0.2f,
 subset: "training",
 seed: seed);

 foreach (var (text_batch, label_batch) in raw_train_ds.take(1))
 {
 foreach (var i in range(3))
 {
 print("Review", text_batch.StringData()[i]);
 print("Label", label_batch.numpy()[i]);
 }
 }

 print("Label 0 corresponds to", raw_train_ds.class_names[0]);
 print("Label 1 corresponds to", raw_train_ds.class_names[1]);

 var raw_val_ds = keras.preprocessing.text_dataset_from_directory(
 train_dir,
 batch_size: batch_size,
 validation_split: 0.2f,
 subset: "validation",
 seed: seed);

 var test_dir = Path.Combine(data_dir, "test");
 var raw_test_ds = keras.preprocessing.text_dataset_from_directory(
 test_dir,
 batch_size: batch_size);

 var max_features = 10000;
 var sequence_length = 250;

 Func<Tensor, Tensor> custom_standardization = input_data =>
 {
 var lowercase = tf.strings.lower(input_data);
 var stripped_html = tf.strings.regex_replace(lowercase, "<br />", " ");
 return tf.strings.regex_replace(stripped_html,
 "'[!\"\\#\\$%\\&\'\\(\\)\\*\\+,\\-\\./:;<=>\\?@\\[\\\\\\]\\^_`\\{\\|\\}\\~]'",
 "");
 };

 var vectorize_layer = keras.layers.preprocessing.TextVectorization(standardize: custom_standardization,
 max_tokens: max_features,
 output_mode: "int",
 output_sequence_length: sequence_length);

 var train_text = raw_train_ds.map(inputs => inputs[0]);
 //vectorize_layer.adapt(train_text);
 }
}