Uses vision AI to classify recyclable materials on the fly C#
👤 Sharing: AI
```csharp
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.Vision;
namespace RecycleClassificationAI
{
// 1. Define Data Structures
/// <summary>
/// Represents the input data for the ML model (an image and its label).
/// </summary>
public class ImageData
{
[LoadColumn(0)] // Tells ML.NET to load data from the 0th column
public string ImagePath { get; set; }
[LoadColumn(1)] // Tells ML.NET to load data from the 1st column
public string Label { get; set; }
}
/// <summary>
/// Represents the image data ready for ML training. Includes the pre-processed image (as a float array) and the label.
/// </summary>
public class ImageInputData
{
[ImageType(224, 224)] // Specify expected image dimensions (for resizing). This is important for many pretrained models.
public Bitmap Image { get; set; }
public string Label { get; set; }
}
/// <summary>
/// Represents the prediction result. Includes the predicted label and a confidence score.
/// </summary>
public class ImagePrediction : ImageData
{
[ColumnName("PredictedLabel")]
public string PredictedLabel { get; set; }
public float[] Score { get; set; }
}
class Program
{
// 2. Define Paths and Constants
private static readonly string BasePath = AppDomain.CurrentDomain.BaseDirectory;
private static readonly string ImageFolder = Path.Combine(BasePath, "images"); // Where the images are located
private static readonly string DataPath = Path.Combine(BasePath, "data", "labels.tsv"); // Path to the labels file (ImagePath\tLabel)
private static readonly string ModelPath = Path.Combine(BasePath, "model", "recycleModel.zip"); // Where the trained model will be saved
// 3. Main Method
static async Task Main(string[] args)
{
// Check if the data directory exists
if (!Directory.Exists(ImageFolder))
{
Console.WriteLine($"Error: Image folder not found: {ImageFolder}. Make sure you have a folder named 'images' in the application directory with images of recyclable items.");
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
return;
}
// Ensure data file exists
if (!File.Exists(DataPath))
{
Console.WriteLine($"Error: Labels file not found: {DataPath}. Make sure you have a labels.tsv file in the data folder.");
Console.WriteLine("The labels.tsv file should be in the format: <ImagePath>\t<Label> per line.");
Console.WriteLine("Example: images/plastic_bottle1.jpg\tPlastic");
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
return;
}
MLContext mlContext = new MLContext(seed: 1); // Creates an ML.NET environment. Seed for reproducibility.
// 4. Load Data
IDataView trainingDataView = mlContext.Data.LoadFromTextFile<ImageData>(
path: DataPath,
hasHeader: false,
separatorChar: '\t');
// 5. Build and Train Model
ITransformer model = await TrainModel(mlContext, trainingDataView);
// 6. Evaluate Model (Optional, but recommended)
EvaluateModel(mlContext, trainingDataView, model);
// 7. Save Model
SaveModel(mlContext, model, ModelPath);
// 8. Use Model for Prediction
await ClassifyImage(mlContext, model);
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
// 5. Build and Train Model
public static async Task<ITransformer> TrainModel(MLContext mlContext, IDataView trainingDataView)
{
Console.WriteLine("=============== Training the model ===============");
var trainingPipeline = mlContext.Transforms.Conversion.MapValueToKey(outputColumnName: "LabelAsKey", inputColumnName: "Label") //convert the text label to a numerical key
.Append(mlContext.Transforms.LoadRawImageBytes(outputColumnName: "Image", imageFolder: ImageFolder)) // Loads the raw image bytes. Needs the ImageFolder setting.
.Append(mlContext.Transforms.ResizeImages(outputColumnName: "Image", imageWidth: 224, imageHeight: 224, inputColumnName: "Image")) // Resizes images to a consistent size. Many models require a specific input size.
.Append(mlContext.Transforms.ExtractPixels(outputColumnName: "PixelValues", inputColumnName: "Image")) // Extracts the pixel data as a float array
.Append(mlContext.MulticlassClassification.Trainers.ImageClassification(featureColumnName: "PixelValues", labelColumnName: "LabelAsKey")
.Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabelAsKey")); // Maps the predicted key back to the original label string
ITransformer model = trainingPipeline.Fit(trainingDataView);
Console.WriteLine("=============== End of training ===============");
return model;
}
// 6. Evaluate Model (Optional, but recommended)
private static void EvaluateModel(MLContext mlContext, IDataView trainingDataView, ITransformer model)
{
Console.WriteLine("=============== Evaluating Model accuracy with Training data===============");
IDataView predictions = model.Transform(trainingDataView);
// Evaluate the model using multi-class classification metrics.
var metrics = mlContext.MulticlassClassification.Evaluate(predictions, labelColumnName: "LabelAsKey", predictedLabelColumnName: "PredictedLabelAsKey");
Console.WriteLine($"LogLoss is: {metrics.LogLoss}");
Console.WriteLine($"PerClassLogLoss is: String.Join(\" , \", metrics.PerClassLogLoss.Select(c => c.ToString()))");
Console.WriteLine($"MacroAccuracy is: {metrics.MacroAccuracy}");
Console.WriteLine($"MicroAccuracy is: {metrics.MicroAccuracy}");
Console.WriteLine($"TopKAccuracy is: {metrics.TopKAccuracy}");
Console.WriteLine("=============== End of model evaluation ===============");
}
// 7. Save Model
public static void SaveModel(MLContext mlContext, ITransformer model, string modelPath)
{
Console.WriteLine("=============== Saving the model to disk ===============");
mlContext.Model.Save(model, trainingDataViewSchema: null, destination: modelPath);
Console.WriteLine($"Model saved to {modelPath}");
}
// 8. Use Model for Prediction
public static async Task ClassifyImage(MLContext mlContext, ITransformer model)
{
Console.WriteLine("=============== Making a prediction ===============");
//Create prediction engine to try on single image
PredictionEngine<ImageInputData, ImagePrediction> predictionEngine = mlContext.Model.CreatePredictionEngine<ImageInputData, ImagePrediction>(model);
// Load a test image
string imagePath = Path.Combine(ImageFolder, "plastic_bottle1.jpg"); // Change this to an image you want to test
if (!File.Exists(imagePath))
{
Console.WriteLine($"Error: Test image not found: {imagePath}. Make sure you have a test image named 'plastic_bottle1.jpg' in the images folder, or change the imagePath variable.");
return;
}
Bitmap bitmap = new Bitmap(Image.FromFile(imagePath));
// Make prediction on single image
var imageInputData = new ImageInputData { Image = bitmap, Label = "" }; // Label is not used for prediction, so can be empty
var prediction = predictionEngine.Predict(imageInputData);
Console.WriteLine($"Image: {Path.GetFileName(imagePath)} predicted as: {prediction.PredictedLabel} with score: {string.Join(" , ", prediction.Score.Select(s => s.ToString("0.00")))}");
}
}
}
```
**Explanation:**
1. **Data Structures:**
* `ImageData`: Defines the structure for raw image data, including the image path and its associated label (e.g., "Plastic," "Glass," "Paper"). This is what's loaded from the `labels.tsv` file.
* `ImageInputData`: Holds the image in a Bitmap format and a label. It prepares the image for ML.NET's transformations, including resizing.
* `ImagePrediction`: Holds the predicted label and an array of confidence scores for each possible class.
2. **Paths and Constants:**
* `BasePath`: Gets the application's base directory.
* `ImageFolder`: Specifies the directory where the images are stored (e.g., `"images"` folder in the application's root). **You MUST create this folder and put your images in it.**
* `DataPath`: Points to the file that lists the images and their labels (e.g., `"data/labels.tsv"`). **You MUST create the `data` folder and the `labels.tsv` file inside it, and fill it with the correct data.**
* `ModelPath`: Defines the location where the trained model will be saved.
3. **Main Method:**
* Creates an `MLContext`: This is the core object for ML.NET operations. The `seed` ensures consistent results across multiple runs (important for debugging and reproducibility).
* Loads the training data using `mlContext.Data.LoadFromTextFile<ImageData>()`. This reads the `labels.tsv` file and creates an `IDataView` (ML.NET's data format). It maps the columns based on the `LoadColumn` attributes in the `ImageData` class.
* Calls `TrainModel()` to build and train the model.
* (Optional) Calls `EvaluateModel()` to assess the model's performance on the training data.
* Saves the trained model to a file using `SaveModel()`.
* Calls `ClassifyImage()` to make a prediction on a sample image using the trained model.
4. **`TrainModel()` Method:**
* Creates a `trainingPipeline`: This is a sequence of data transformations and the training algorithm.
* `MapValueToKey`: Converts the text labels (e.g., "Plastic") into numerical keys that the ML algorithm can work with.
* `LoadRawImageBytes`: Loads the raw image bytes from the specified `ImageFolder`.
* `ResizeImages`: Resizes the images to a consistent size (224x224 in this example). **Many pre-trained models require specific image dimensions!**
* `ExtractPixels`: Extracts the pixel data from the images and converts it into a float array. This array becomes the input feature for the model.
* `ImageClassification`: This is the core training algorithm specifically designed for image classification tasks. It uses the pixel data (`PixelValues`) to predict the label (`LabelAsKey`). The ImageClassification trainer uses a pre-trained neural network.
* `MapKeyToValue`: Converts the predicted numerical key back into the original text label (e.g., "Plastic").
* `trainingPipeline.Fit(trainingDataView)`: Trains the model using the loaded data and the defined pipeline.
5. **`EvaluateModel()` Method:**
* Transforms the training data using the trained model to get predictions.
* Uses `mlContext.MulticlassClassification.Evaluate()` to calculate metrics such as LogLoss, MacroAccuracy, and MicroAccuracy. These metrics provide insights into the model's performance.
6. **`SaveModel()` Method:**
* Saves the trained model to a file using `mlContext.Model.Save()`.
7. **`ClassifyImage()` Method:**
* Creates a `PredictionEngine<ImageInputData, ImagePrediction>`: This object is used to make predictions on single images.
* Loads a test image (e.g., `"plastic_bottle1.jpg"`). **Change this to the path of an image you want to classify.**
* Creates an `ImageInputData` object with the test image.
* `predictionEngine.Predict()`: Uses the model to predict the label of the image.
* Prints the predicted label and the confidence scores.
**How to Use:**
1. **Create a New C# Console Application:** In Visual Studio (or your preferred IDE), create a new C# Console Application project.
2. **Install Required NuGet Packages:**
* Right-click on your project in the Solution Explorer and select "Manage NuGet Packages...".
* Search for and install the following packages:
* `Microsoft.ML`
* `Microsoft.ML.ImageAnalytics`
* `Microsoft.ML.Vision` (This package brings in some default dependencies, but depending on your specific needs you may need additional packages)
* `System.Drawing.Common` (You might need to add this explicitly for Bitmap/Image support on some platforms)
3. **Replace the Code:** Copy and paste the C# code above into your `Program.cs` file.
4. **Prepare Your Data:**
* **Create the `images` folder:** In your project's directory (the same level as your `.csproj` file), create a folder named `images`.
* **Gather Images:** Collect images of recyclable materials (e.g., plastic bottles, glass bottles, cardboard boxes, paper). Place these images in the `images` folder. It's crucial to have a good number of images per class (at least 50-100 per class is recommended for decent accuracy).
* **Create the `data` folder:** Create a folder named `data` in the same directory as the `images` folder.
* **Create the `labels.tsv` file:** Inside the `data` folder, create a text file named `labels.tsv`. This file should contain a list of your image paths and their corresponding labels, separated by a tab character (`\t`).
Here's an example `labels.tsv` file:
```
images/plastic_bottle1.jpg Plastic
images/plastic_bottle2.jpg Plastic
images/plastic_bottle3.jpg Plastic
images/glass_bottle1.jpg Glass
images/glass_bottle2.jpg Glass
images/paper1.jpg Paper
images/paper2.jpg Paper
images/cardboard1.jpg Cardboard
images/cardboard2.jpg Cardboard
```
**Important:**
* The image paths in `labels.tsv` should be relative to the application's base directory. In this example, they're relative to the root of your project.
* Make sure the labels (e.g., "Plastic," "Glass," "Paper," "Cardboard") are consistent.
* Use tab characters (`\t`) as separators, **not spaces**.
5. **Create the `model` folder:** In your project's directory, create a folder named `model`. This is where the trained model will be saved.
6. **Configure the `plastic_bottle1.jpg` Test Image:** Make sure there is an image called `plastic_bottle1.jpg` in your `images` folder. The code looks for this file to make a prediction. Change the `imagePath` variable in the `ClassifyImage` method if you want to use a different test image. If you are testing with an image that has a different extension (e.g., .png, .jpeg), be sure to update the code to reflect that.
7. **Run the Application:** Build and run your C# console application. It will:
* Load the data from `labels.tsv`.
* Train the model using the images in the `images` folder.
* Evaluate the model.
* Save the trained model to `model/recycleModel.zip`.
* Make a prediction on `plastic_bottle1.jpg`.
* Print the predicted label and confidence score.
**Troubleshooting:**
* **"Could not find file" errors:** Double-check the paths in your code (especially `ImageFolder`, `DataPath`, and `ModelPath`) and make sure the files and folders exist in the specified locations.
* **Low Accuracy:**
* **Insufficient Data:** Image classification models often require a large amount of training data to achieve good accuracy. Make sure you have enough images (at least 50-100) per class.
* **Poor Image Quality:** Clear, well-lit images are important. Blurry or poorly lit images can reduce accuracy.
* **Inconsistent Labeling:** Make sure your labels are consistent and accurate.
* **Data Augmentation:** Consider using data augmentation techniques (e.g., rotating, cropping, flipping, changing brightness) to artificially increase the size of your training dataset. ML.NET has transforms that can help with this.
* **System.Drawing.Common issues:** If you encounter errors related to `System.Drawing.Common`, especially on non-Windows platforms, you might need to install the package explicitly and configure it correctly in your project.
* **Model Loading Errors:** If you receive an exception when trying to load your saved model (e.g. if you try to load it in a separate application), make sure you have all the necessary ML.NET NuGet packages installed in the project that *loads* the model, not just the project that created it.
This complete example provides a solid foundation for building a recycle classification system using vision AI in C#. Remember that accuracy heavily depends on the quality and quantity of your training data. You can improve the model's performance by:
* Adding more training images.
* Experimenting with different ML.NET transforms.
* Trying different image classification trainers within ML.NET.
👁️ Viewed: 2
Comments