# James Kleckner, Julian Reynolds

# CSCI 507

# Final Project: Webcam Painting

import cv2

import numpy as np

from os.path import exists

def main():

# initialize webcam capture

cam_feed = cv2.VideoCapture(0)

IMAGE_WIDTH = int(cam_feed.get(3))

IMAGE_HEIGHT = int(cam_feed.get(4))

# camera properties:

fx = 400

fy = 400

cx = 240

cy = 320

K = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]])

# ArUco dimension in inches

MARKER_LENGTH = 4

# create ArUco tag dictionary

arucoDict = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_4X4_50)

# define threshold for acceptable distance between centroids

CENTROID_THRESH = 0.5

DRAWING_THRESH = 1.7 * IMAGE_WIDTH

# lower edge of user interface

INTERFACE_Y = 120

can_select = True

erasing = False

drawing_radius = 5

erasing_radius = 15

current_tool = 3

is_displaying = False

is_saving = False

# previous cursor location, this will be used to check for extraneous centroids being reported

cursor = (0, 0)

cursor_previous = cursor

frame_count = 0

cursor_lost = frame_count

first_loop_display = True

first_loop_save = True

drawing = np.full(shape=(IMAGE_HEIGHT, IMAGE_WIDTH, 3), fill_value=([255, 255, 255]), dtype=np.uint8)

# VIDEO LOOP

# ==================================================================================================================

while True:

if cursor_lost >= 10:

frame_count = 0

cursor_lost = 0

print("Reset Cursor")

canvas = np.full(shape=(IMAGE_HEIGHT, IMAGE_WIDTH, 3), fill_value=([0, 0, 0]), dtype=np.uint8)

canvas += drawing

check, frame = cam_feed.read()

# flip camera input horizontally so that it is not confusing for users when painting

frame = cv2.flip(frame, 1)

# convert to grayscale

grayscale = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

#############

white_centroids, centroids_white_stats, black_centroids, centroids_black_stats = find_centroids(grayscale)

#############

white_sizes = centroids_white_stats[:, -1]

black_sizes = centroids_black_stats[:, -1]

for i, white_centroid in enumerate(white_centroids):

for black_centroid in black_centroids:

distance = np.linalg.norm(white_centroid - black_centroid)

if distance < CENTROID_THRESH:

x = int(np.mean([white_centroid[0], black_centroid[0]]))

y = int(np.mean([white_centroid[1], black_centroid[1]]))

cv2.drawMarker(grayscale, position=(x, y),

color=(0, 0, 255), markerType=cv2.MARKER_CROSS, thickness=1)

cursor = [x, y]

if frame_count == 0:

cursor_previous = cursor

else:

if get_distance(cursor, cursor_previous) < 20:

if white_sizes[i] > DRAWING_THRESH:

cv2.drawMarker(canvas, position=(int(cursor[0]),

int(cursor[1])), color=(0, 0, 255),

markerType=cv2.MARKER_CROSS, thickness=2)

# DRAWING LOGIC BELOW

if not erasing:

current_color = get_current_color(current_tool, return_string=False)

cv2.circle(drawing, center=(int(cursor[0]), int(cursor[1])), radius=drawing_radius,

color=current_color,

thickness=-1)

else:

cv2.circle(drawing, center=(int(cursor[0]), int(cursor[1])), radius=erasing_radius,

color=(255, 255, 255),

thickness=-1)

else:

cv2.drawMarker(canvas, position=(int(cursor[0]),

int(cursor[1])), color=(0, 0, 0),

markerType=cv2.MARKER_DIAMOND, thickness=2)

cursor_previous = cursor

else:

cursor_lost += 1

# DRAWING USER INTERFACE -------------------------------------------------------------------------------

# change color option

cv2.rectangle(canvas, pt1=(0, 60), pt2=(105, INTERFACE_Y), color=(150, 150, 150), thickness=cv2.FILLED)

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1.3, thickness=2, text="Change",

color=(255, 255, 255), org=(10, 85))

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1.3, thickness=2, text="Color",

color=(255, 255, 255), org=(20, 105))

# eraser option

if erasing:

eraser_color = (100, 100, 100)

eraser_text_color = (0, 255, 255)

else:

eraser_color = (150, 150, 150)

eraser_text_color = (255, 255, 255)

cv2.rectangle(canvas, pt1=(108, 60), pt2=(208, INTERFACE_Y), color=eraser_color, thickness=cv2.FILLED)

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=2, thickness=2, text="Erase",

color=eraser_text_color, org=(113, 100))

# save drawing option

cv2.rectangle(canvas, pt1=(211, 60), pt2=(311, INTERFACE_Y), color=(150, 150, 150), thickness=cv2.FILLED)

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1.3, thickness=2, text="Save",

color=(255, 255, 255), org=(235, 85))

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1.3, thickness=2, text="Drawing",

color=(255, 255, 255), org=(220, 105))

# display drawing option

cv2.rectangle(canvas, pt1=(314, 60), pt2=(414, INTERFACE_Y), color=(150, 150, 150), thickness=cv2.FILLED)

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1.3, thickness=2, text="Display",

color=(255, 255, 255), org=(327, 85))

cv2.putText(canvas, fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1.3, thickness=2, text="Drawing",

color=(255, 255, 255), org=(322, 105))

if cursor[1] < INTERFACE_Y and can_select:

if cursor[0] < 105:

current_tool += 1

if current_tool > 10:

current_tool = 0

elif cursor[0] < 208:

if erasing:

erasing = False

else:

erasing = True

elif cursor[0] < 311:

is_saving = True

elif cursor[0] < 414:

is_displaying = True

can_select = False

if cursor[1] > INTERFACE_Y:

can_select = True

# current color mode text

cv2.putText(canvas, fontScale=1.5, fontFace=cv2.FONT_HERSHEY_PLAIN, thickness=2, text="Current Tool:",

org=(IMAGE_WIDTH - 300, 40), color=(0, 0, 0))

if erasing:

current_tool_string = "Erasing"

else:

current_tool_string = get_current_color(current_tool, return_string=True)

cv2.putText(canvas, fontScale=1.5, fontFace=cv2.FONT_HERSHEY_PLAIN, thickness=2, text=current_tool_string,

org=(IMAGE_WIDTH - 120, 40), color=(0, 0, 0))

# -------------------------------------------------------------------------------------------------------

resize_display = True

if is_displaying:

frame = cv2.flip(frame, 1)

corners, ids, _ = cv2.aruco.detectMarkers(image=frame, dictionary=arucoDict)

if ids is not None:

drawing_exists = exists("drawing_" + str(ids[0][0]) + ".jpg")

if not drawing_exists:

frame = cv2.flip(frame, 1)

cv2.rectangle(frame, pt1=(0, 60), pt2=(IMAGE_WIDTH, 120), color=(200, 200, 200),

thickness=cv2.FILLED)

cv2.putText(frame, text="ERROR: No Drawing for this Tag!", org=(IMAGE_WIDTH // 8, 100),

fontFace=cv2.FONT_HERSHEY_PLAIN,

fontScale=1.75,

thickness=2, color=(0, 0, 255))

frame = cv2.flip(frame, 1)

else:

filename = "drawing_" + str(ids[0][0]) + ".jpg"

drawing = cv2.imread(filename)

try:

drawing = cv2.resize(drawing, (640, 480))

except:

print('Stupid resize error')

dist_coeff = None

aruco_rvecs, aruco_tvecs, _ = cv2.aruco.estimatePoseSingleMarkers(corners=corners,

markerLength=MARKER_LENGTH,

cameraMatrix=K.astype("float32"),

distCoeffs=dist_coeff)

# get translation and rotation vectors of tags

aruco_rvec = aruco_rvecs[0]

aruco_tvec = aruco_tvecs[0]

aruco_rmat = cv2.Rodrigues(aruco_rvec)[0]

width = drawing.shape[0]

height = drawing.shape[1]

drawing = cv2.flip(drawing, 1)

# Add image onto white paper using homography:

# set of points that define the corners of the CCC targets

aruco_corners = np.array([corners[0][0][1], corners[0][0][2], corners[0][0][3], corners[0][0][0]],

dtype=np.int32)

# coordinates of picture corner locations on paper relative to center of ArUco tag

paper_coordinates = np.array([[5.5, 4.25, 0], [5.5, -4.25, 0], [-5.5, -4.25, 0], [-5.5, 4.25, 0]])

# tag coordinates to camera coordinates

H_t_c = np.block([[aruco_rmat, aruco_tvec.T]])

pts0 = np.empty(shape=(4, 2), dtype=np.int32)

for i, point in enumerate(paper_coordinates):

P_w = np.append(point, [1])

Mext = H_t_c[0:3, :]

p = K @ Mext @ P_w

p = p / p[2]

pts0[i, 0] = p[0]

pts0[i, 1] = p[1]

# set of points that define the corners of the image we want to put on top

pts1 = np.array([(height, 0), (height, width), (0, width), (0, 0)])

H, _ = cv2.findHomography(pts1, pts0)

bgr_output = cv2.warpPerspective(drawing, H, (IMAGE_WIDTH, IMAGE_HEIGHT))

cv2.fillConvexPoly(frame, pts0, color=[0, 0, 0])

frame = fuse_color_images(frame, bgr_output)

if first_loop_display:

cv2.destroyAllWindows()

first_loop_display = False

frame = cv2.flip(frame, 1)

cv2.imshow("Drawing Display", frame)

key = cv2.waitKey(1)

if key == 27:

break

elif is_saving:

if first_loop_save:

cv2.destroyAllWindows()

first_loop_save = False

cv2.rectangle(frame, pt1=(0, 60), pt2=(IMAGE_WIDTH, 120), color=(200, 200, 200),

thickness=cv2.FILLED)

cv2.putText(frame, text="Display ArUco Paper", org=(IMAGE_WIDTH // 5, 100),

fontFace=cv2.FONT_HERSHEY_PLAIN,

fontScale=2.2,

thickness=2, color=(0, 0, 0))

frame = cv2.flip(frame, 1)

corners, ids, _ = cv2.aruco.detectMarkers(image=frame, dictionary=arucoDict)

if ids is not None:

# cv2.aruco.drawDetectedMarkers(image=frame, corners=corners, ids=ids, borderColor=(0, 0, 255))

filename = "drawing_" + str(ids[0][0]) + ".jpg"

cv2.imwrite(filename, drawing)

is_saving = False

is_displaying = True

frame = cv2.flip(frame, 1)

cv2.imshow("Save Image", frame)

key = cv2.waitKey(1)

if key == 27:

break

else:

cv2.imshow("Paint", canvas)

cv2.imshow("Me", grayscale)

key = cv2.waitKey(1)

if key == 27:

break

frame_count += 1

# ==================================================================================================================

cv2.destroyAllWindows()

def get_distance(point1, point2): # this function handles the right triangle math for two points

distance_x = point1[1] - point2[1]

distance_y = point1[0] - point2[0]

distance = (distance_y ** 2 + distance_x ** 2) ** 0.5

return distance

def get_current_color(current_tool, return_string):

if current_tool == 0:

# brown

color_string = "Brown"

color = (115, 69, 29)

elif current_tool == 1:

# red

color_string = "Red"

color = (0, 0, 255)

elif current_tool == 2:

# orange

color_string = "Orange"

color = (255, 115, 0)

elif current_tool == 3:

# yellow

color_string = "Yellow"

color = (0, 255, 255)

elif current_tool == 4:

# green

color_string = "Green"

color = (0, 255, 0)

elif current_tool == 5:

# blue

color_string = "Blue"

color = (255, 0, 0)

elif current_tool == 6:

# indigo

color_string = "Indigo"

color = (51, 0, 29)

elif current_tool == 7:

# violet

color_string = "Violet"

color = (168, 77, 168)

elif current_tool == 8:

# pink

color_string = "Pink"

color = (80, 75, 100)

elif current_tool == 9:

# black

color_string = "Black"

color = (0, 0, 0)

elif current_tool == 10:

# grey

color_string = "Grey"

color = (100, 100, 100)

if return_string:

return color_string

else:

return color

def fuse_color_images(A, B):

assert (A.ndim == 3 and B.ndim == 3)

assert (A.shape == B.shape)

# Allocate result

C = np.zeros(A.shape, dtype=np.uint8)

# Create masks for pixels that are not zero.

A_mask = np.sum(A, axis=2) > 0

B_mask = np.sum(B, axis=2) > 0

# Compute regions of overlap.

A_only = A_mask & ~B_mask

B_only = B_mask & ~A_mask

A_and_B = A_mask & B_mask

C[A_only] = A[A_only]

C[B_only] = B[B_only]

C[A_and_B] = 1.0 * A[A_and_B] + 1.0 * B[A_and_B]

return C

def find_centroids(gray_image):

"""

Finds the centroids of gray and black blobs in the image frame.

"""

gray_smooth = cv2.GaussianBlur(gray_image, ksize=(1, 1), sigmaY=1, sigmaX=1)

_, binary_img = cv2.threshold(gray_smooth, thresh=0, maxval=255, type=cv2.THRESH_BINARY + cv2.THRESH_OTSU)

kernel = np.ones((5, 5), np.uint8)

binary = cv2.morphologyEx(binary_img, cv2.MORPH_CLOSE, kernel)

binary = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel)

_, _, black_stats, black_centroids = cv2.connectedComponentsWithStats(binary)

# Invert image

invert_image = cv2.bitwise_not(binary)

_, _, white_stats, white_centroids = cv2.connectedComponentsWithStats(invert_image)

return white_centroids, white_stats, black_centroids, black_stats

if __name__ == "__main__":

main()