# You must make sure to run all cells in sequence using shift + enter or you might encounter errors
from pykubegrader.initialize import initialize_assignment
responses = initialize_assignment("1_angry_birds_image", "week_9", "lab", assignment_points = 105.0, assignment_tag = 'week9-lab')
# Initialize Otter
import otter
grader = otter.Notebook("1_angry_birds_image.ipynb")
Angry Birds Come Back!#
In a previous assignment we learned how to shoot birds at pigs with unerring accuracy. Now we want to make the game even easier by using image processing to detect scene elements. This is much akin to any computer vision task in science, engineering, manufacturing and medicine.
In this assignment, we will use image processing to detect the main structure of the scene, and then use that mask to isolate the different elements in the scene.
We will walk you through the steps of the process.
Note: You are now getting to be more of a pro at Python. While we do not expect you to know all the syntax off the top of your head, we do expect you to be able to do a little digging and figure out how to do what you need.
Importing packages#
# We nede to import the following packages:
# - numpy using common alias np
# - matplotlib.pyplot using common alias plt
# - from skimage -- io, color, filters, feature, morphology, measure, segmentation, label, regionprops, clear_border -- note some of these are sub-packages[]
...
grader.check("import_packages")
# We need to load the image and convert it to grayscale, to load the image assign a variable to the image path, to the name of the image "angry-bird.webp"
...
# load the image using the io.imread function, assign the result to the variable image
...
# Convert to grayscale for edge detection
# use the color.rgb2gray function to convert the image to grayscale, assign the result to the variable gray_image
...
# use the plt.subplots function to create a figure and a single axis, assign the result to the variables fig and ax
...
# use the ax.imshow function to display the image
...
# use the ax.set_title function to set the title of the image - set the title to "Original Image"
...
# use the ax.axis function to turn off the axis labels
...
# use the plt.show function to display the image
...
grader.check("load-and-visualize-image")
# Apply Canny edge detector to detect edges in the grayscale image
# use the feature.canny function to detect the edges in the image, assign the result to the variable edges
# set the sigma to 2.0
...
# wow I guess I lied when I said we could do this in just a few lines! it is 1 line!
# Visualize the edges
# use the plt.subplots function to create a figure with 2 subplots, assign the result to the variables fig and ax
# the figures should be 10 inches wide and 5 inches tall
# the first subplot should display the grayscale image
# the second subplot should display the edges
# you want the figures layed out in a row with 2 columns
...
# use the ax.imshow function to display the grayscale image in the first subplot -- hint you need to use indexing to get the first subplot
# set the colormap to "gray"
...
# use the ax.set_title function to set the title of the image - set the title to "Grayscale Image"
...
# use the ax.axis function to turn off the axis labels
...
# use the ax.imshow function to display the edges in the second subplot -- hint you need to use indexing to get the second subplot
# set the colormap to "gray"
...
# use the ax.set_title function to set the title of the image - set the title to "Edges Detected (Canny)"
...
# use the ax.axis function to turn off the axis labels
...
# use the plt.show function to display the image
...
grader.check("edge-detection")
# Dilate the edges to connect broken fragments
# use the morphology.dilation function to dilate the edges, assign the result to the variable dilated_edges
# use the morphology.disk function to create a disk-shaped structuring element with a radius of 3
...
# Fill holes inside the detected edges to create a solid mask
# use the morphology.remove_small_holes function to fill the holes in the edges, assign the result to the variable filled_mask, a threshold of 5000 is used
...
# use the morphology.remove_small_objects function to remove the small objects in the mask, assign the result to the variable filled_mask, a minimum size of 60000 is used
...
# use the plt.subplots function to create a figure with 1 subplot, assign the result to the variables fig and ax
# the figure should be 6 inches wide and 4 inches tall
...
# use the ax.imshow function to display the filled mask
...
# use the ax.set_title function to set the title of the image - set the title to "Filled Mask"
...
# use the ax.axis function to turn off the axis labels
...
# use the plt.show function to display the image
...
grader.check("refinement-dilation-and-hole-filling")
# use the image.copy function to create a copy of the image, assign the result to the variable structure_only
...
# use the ~ operator to invert the mask, assign the result to the variable inverted_mask
...
# use the structure_only variable to set the background to white, assign the result to the variable structure_only
...
# use the plt.subplots function to create a figure with 1 subplot, assign the result to the variables fig and ax
# the figure should be 6 inches wide and 4 inches tall
...
# use the ax.imshow function to display the structure_only image
...
# use the ax.set_title function to set the title of the image - set the title to "Main Structure Isolated"
...
# use the ax.axis function to turn off the axis labels
...
# use the plt.show function to display the image
...
grader.check("isolate-main-structure")
# Build the base class for the image processor called ImageProcessor
...
# initialize the image processor class with the image to analyze
...
# assign the image to the image attribute
...
# convert the image to HSV color space, assign the result to the hsv_image attribute
...
# convert the image to grayscale, assign the result to the gray_image attribute
...
# create an empty dictionary to store the masks, labels_dict and regions_dict to their respective attributes
...
# define the display_image method, look at the docstring for information on the inputs and outputs
...
"""
Display the image with a given title.
Parameters:
title (str): The title of the image to be displayed. Default is "Image".
"""
...
grader.check("image-processor")
Question 6 (Points: 14.0): Color Mask Class#
Now we want to make sure that we can isloate the different objects in the image based on the color. We will use the ColorMask class to help us with this. The color mask class will identify the pixels in the image that fall within a certain range of hue, saturation, and value (brightness) values - and then remove the pixels that are not part of contiguous regions.
# build the ColorMask class, look at the docstring for information on the inputs and outputs, and the methods
...
# define the __init__ method, look at the docstring for information on the inputs and outputs
...
"""
Initialize the ColorMask object with HSV image and range values.
Parameters:
hsv_img (numpy.ndarray): The HSV image to process.
hue_range (tuple): The range of hue values to include in the mask.
sat_range (tuple): The range of saturation values to include in the mask.
val_range (tuple): The range of value (brightness) values to include in the mask.
"""
# assign all variables to the respective attributes, add an attribute for the mask initialized to None
...
# define the build_mask method, look at the docstring for information on the inputs and outputs
...
"""
Create a binary mask for pixels whose HSV values fall within the
specified hue, saturation, and value ranges.
Returns:
numpy.ndarray: A binary mask where pixels within the specified ranges are True.
"""
# extract the hue, saturation, and value channels from the HSV image as H, S, V
# The format of the hsv_img is (x,y,3), where x is the height and y is the width,
# and the 3 is for the 3 color channels of the image
...
# create a binary mask where pixels within the specified ranges are True
# this is when for example H >= self.hue_range[0] and H <= self.hue_range[1], remember to use the binary & operator
# you want to ensure all the color conditions are met for the mask to be True
...
# return the mask
return self.mask
# define the clean_mask method, look at the docstring for information on the inputs and outputs
...
"""
Apply morphological operations to clean the binary mask.
Returns:
numpy.ndarray: The cleaned binary mask after applying closing and opening operations.
"""
# apply morphological closing and opening operations to the mask
# use the morphology.closing function to close the mask, use a disk-shaped structuring element with a radius of 3
# use the morphology.opening function to open the mask, use a disk-shaped structuring element with a radius of 3
...
# return the mask
...
grader.check("Color-mask-and-merge")
Question 7 (Points: 13.0): Derived Classes for Specific Color Masks#
Now we want to create the derived classes for the specific color masks. We will create a class for the pig mask, the wood mask, the gray mask, and the blue mask. This is the power of using classes, one class can be used to create multiple derived classes, and each derived class can have its own unique properties and methods.
# build the PigMask class, look at the docstring for information on the inputs and outputs
...
"""
A class to represent a mask for detecting pig objects in an image.
Inherits from the ColorMask base class and initializes the mask with
specific HSV color range values that correspond to the color of pigs
in the image.
"""
# initialize the PigMask class with the hsv_img, hue_range, sat_range, and val_range
...
"""
Constructs all the necessary attributes for the PigMask object.
Parameters
----------
hsv_img : ndarray
The HSV image in which the pig mask is to be applied.
hue_range : tuple, optional
The range of hue values for the pig mask (default is (0.20, 0.40)).
sat_range : tuple, optional
The range of saturation values for the pig mask (default is (0.50, 1.00)).
val_range : tuple, optional
The range of value (brightness) values for the pig mask (default is (0.30, 1.00)).
"""
# initialize the ColorMask class with the hsv_img, hue_range, sat_range, and val_range
...
# build the WoodMask class, look at the docstring for information on the inputs and outputs
...
"""
A class to represent a mask for detecting wood objects in an image.
Inherits from the ColorMask base class and initializes the mask with
specific HSV color range values that correspond to the color of wood
in the image.
"""
# initialize the WoodMask class with the hsv_img, hue_range, sat_range, and val_range
...
"""
Constructs all the necessary attributes for the WoodMask object.
Parameters
----------
hsv_img : ndarray
The HSV image in which the wood mask is to be applied.
hue_range : tuple, optional
The range of hue values for the wood mask (default is (0.04, 0.15)).
sat_range : tuple, optional
The range of saturation values for the wood mask (default is (0.40, 1.00)).
val_range : tuple, optional
The range of value (brightness) values for the wood mask (default is (0.30, 1.00)).
"""
# initialize the ColorMask class with the hsv_img, hue_range, sat_range, and val_range
...
# build the GrayMask class, look at the docstring for information on the inputs and outputs
...
"""
A class to represent a mask for detecting gray objects in an image.
Inherits from the ColorMask base class and initializes the mask with
specific HSV color range values that correspond to the color of gray
objects in the image.
"""
# initialize the GrayMask class with the hsv_img, hue_range, sat_range, and val_range
...
"""
Constructs all the necessary attributes for the GrayMask object.
Parameters
----------
hsv_img : ndarray
The HSV image in which the gray mask is to be applied.
hue_range : tuple, optional
The range of hue values for the gray mask (default is (0.00, 0.833)).
sat_range : tuple, optional
The range of saturation values for the gray mask (default is (0.00, 0.52)).
val_range : tuple, optional
The range of value (brightness) values for the gray mask (default is (0.30, 0.89)).
"""
# initialize the ColorMask class with the hsv_img, hue_range, sat_range, and val_range
...
# build the BlueMask class, look at the docstring for information on the inputs and outputs
...
"""
A class to represent a mask for detecting blue objects in an image.
Inherits from the ColorMask base class and initializes the mask with
specific HSV color range values that correspond to the color of blue
objects in the image.
"""
# initialize the BlueMask class with the hsv_img, hue_range, sat_range, and val_range
...
"""
Constructs all the necessary attributes for the BlueMask object.
Parameters
----------
hsv_img : ndarray
The HSV image in which the blue mask is to be applied.
hue_range : tuple, optional
The range of hue values for the blue mask (default is (0.52, 0.57)).
sat_range : tuple, optional
The range of saturation values for the blue mask (default is (0.141, 0.763)).
val_range : tuple, optional
The range of value (brightness) values for the blue mask (default is (0.867, 1.00)).
"""
# initialize the ColorMask class with the hsv_img, hue_range, sat_range, and val_range
...
grader.check("PigMask-main")
# Now we want to build the MaskBuilder class, look at the docstring for information on the inputs and outputs
...
# define the __init__ method, look at the docstring for information on the inputs and outputs
...
"""
Initializes the MaskBuilder with an image.
Parameters
----------
image : ndarray
The input image to be processed. It is expected to be in a format
that can be converted to HSV for mask building.
categories : list of str, optional
A list of categories for which masks will be built. Default is
["pig", "wood", "gray", "blue"].
Attributes
----------
categories : list of str
A list of categories for which masks will be built.
"""
# initialize the ImageProcessor class with the image, as an inheritance
...
# assign the categories to the attributes
...
# define the build_and_clean_masks method, look at the docstring for information on the inputs and outputs
...
"""
Builds and cleans masks for all categories.
This method creates masks for each category defined in `self.categories`
using their respective mask classes. It then cleans each mask and stores
the result in `self.masks`.
The masks are built and cleaned in the order they appear in `self.categories`.
Each mask is stored in `self.masks` with the category name as the key.
If a mask cannot be built, `None` is stored for that category.
"""
# iterate over each category in self.categories assign to the local variable category
...
# get the mask class from the globals
# this is advanced python, we are using the globals() function to get the mask class from the globals
# the f-string is used to format the category name to the correct mask class name
# for example, if the category is "pig", the mask class will be "PigMask"
mask_class = globals().get(f"{category.capitalize()}Mask")
# if the mask class exists, create an instance of the mask class
...
# assign the mask instance to the masks dictionary with the category name as the key
# if the mask_instance.build_mask() is not None, then we can clean the mask, and save the output to the masks dictionary
# otherwise, we save None to the masks dictionary
...
# define the label_and_extract_regions method, look at the docstring for information on the inputs and outputs
...
"""
Labels the cleaned masks and extracts region properties.
This method iterates over each mask in `self.masks`, labels the mask,
and extracts region properties using `regionprops`. The labels and
region properties are stored in `self.labels_dict` and `self.regions_dict`
respectively, with the category name as the key.
For each category, the number of detected regions is printed to the console.
"""
# iterate over each category in self.masks assign to the local variable cat, and mask --hint use the items() method
...
# label the mask using the label function from skimage.measure
# assign the output to the local variable lbl
...
# extract the region properties using the regionprops function from skimage.measure
# assign the output to the local variable regs
...
# assign the labels and regions to the labels_dict and regions_dict attributes, with the category name as the key
...
# print the number of regions detected for the category, the format is "[INFO] Detected <lenght(regs)> regions for <category>"
...
grader.check("MaskBuilder-main-for-pig-wood-gray-blue")
# Build the Visualizer class
...
# Define the display_masks method, which takes in the masks and categories, add static to the method as a decorator
...
"""
Display the masks for all categories.
"""
# Create a figure and a set of subplots
...
# Iterate over the categories and display the masks
...
# Display the mask
...
# Set the title of the mask
...
# Turn off the axis
...
# Show the plot
...
# Define the display_overlays method, which takes in the labels_dict, gray_image, and categories. Add static to the method as a decorator
...
"""
Display overlays for each category.
"""
# Create a figure and a set of subplots
...
# Iterate over the categories and display the overlays
...
# Create the overlay
...
# Display the overlay
...
# Set the title of the overlay
...
# Turn off the axis
...
# Show the plot
...
# Define the display_bounding_boxes method, which takes in the image, regions_dict, and categories. Add static to the method as a decorator
...
"""
Display bounding boxes on the original image.
"""
# Create a color map for the categories as a dictionary as pig-> red, wood-> orange, gray-> gray, blue-> blue
...
# Create a figure and a set of subplots
...
# Display the image
...
# Set the title of the image
...
# Iterate over the categories and display the bounding boxes
...
# Get the regions for the category
...
# Get the color for the category
...
# Iterate over the regions and display the bounding boxes
...
# Check if the region area is greater than 50
...
# Get the bounding box for the region
...
# Create a rectangle for the bounding box
...
# Add the rectangle to the plot
...
# Turn off the axis
...
# Show the plot
...
grader.check("Visualizer-main")
# image = io.imread("angry_birds_level.png") # Replace with your local filename
image = structure_only # Example image variable
# Initialize Mask Builder
mask_builder = MaskBuilder(image)
# Display Original Image
mask_builder.display_image(title="Original Image")
# Build and Clean Masks
mask_builder.build_and_clean_masks()
# Label and Extract Regions
mask_builder.label_and_extract_regions()
# Display Results
Visualizer.display_masks(mask_builder.masks, mask_builder.categories)
Visualizer.display_overlays(mask_builder.labels_dict, mask_builder.gray_image, mask_builder.categories)
Visualizer.display_bounding_boxes(image, mask_builder.regions_dict, mask_builder.categories)
grader.check("writing-main")
Submitting Assignment#
Please run the following block of code using shift + enter to submit your assignment, you should see your score.
from pykubegrader.submit.submit_assignment import submit_assignment
submit_assignment("week9-lab", "1_angry_birds_image")