From d6a91e811ae21673a13476ace853955d5896d619 Mon Sep 17 00:00:00 2001 From: Emmanuel Bertrand Date: Mon, 9 Dec 2019 11:13:28 -0800 Subject: [PATCH 1/2] stop tracking .env --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ec0d5e6..a878628 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files +*.env *.suo *.user *.userosscache From 33dcda6c83d8245503a74fd288f7199952665cad Mon Sep 17 00:00:00 2001 From: Emmanuel Bertrand Date: Mon, 9 Dec 2019 21:30:46 -0800 Subject: [PATCH 2/2] using latest custom vision image --- modules/CameraCapture/app/CameraCapture.py | 197 ++++++------- modules/CameraCapture/module.json | 2 +- .../ImageClassifierService/amd64.Dockerfile | 14 +- modules/ImageClassifierService/app/app.py | 37 ++- modules/ImageClassifierService/app/predict.py | 274 ++++++++++++------ .../ImageClassifierService/arm32v7.Dockerfile | 41 +-- .../build/amd64-requirements.txt | 3 - .../build/arm32v7-requirements.txt | 4 - modules/ImageClassifierService/module.json | 2 +- modules/SenseHatDisplay/app/MessageParser.py | 10 +- modules/SenseHatDisplay/app/__init__.py | 0 modules/SenseHatDisplay/app/main.py | 2 +- modules/SenseHatDisplay/module.json | 2 +- modules/SenseHatDisplay/test/UnitTests.py | 13 +- 14 files changed, 323 insertions(+), 278 deletions(-) delete mode 100644 modules/ImageClassifierService/build/amd64-requirements.txt delete mode 100644 modules/ImageClassifierService/build/arm32v7-requirements.txt create mode 100644 modules/SenseHatDisplay/app/__init__.py diff --git a/modules/CameraCapture/app/CameraCapture.py b/modules/CameraCapture/app/CameraCapture.py index 9715a40..6e5089f 100644 --- a/modules/CameraCapture/app/CameraCapture.py +++ b/modules/CameraCapture/app/CameraCapture.py @@ -1,33 +1,33 @@ -# To make python 2 and python 3 compatible code +#To make python 2 and python 3 compatible code from __future__ import division from __future__ import absolute_import -# Imports -from ImageServer import ImageServer -import ImageServer -from AnnotationParser import AnnotationParser -import AnnotationParser -from VideoStream import VideoStream -import VideoStream -import time -import json -import requests -import numpy +#Imports import sys -if sys.version_info[0] < 3: # e.g python version <3 +if sys.version_info[0] < 3:#e.g python version <3 import cv2 else: import cv2 from cv2 import cv2 # pylint: disable=E1101 # pylint: disable=E0401 -# Disabling linting that is not supported by Pylint for C extensions such as OpenCV. See issue https://github.com/PyCQA/pylint/issues/1955 +# Disabling linting that is not supported by Pylint for C extensions such as OpenCV. See issue https://github.com/PyCQA/pylint/issues/1955 +import numpy +import requests +import json +import time +import VideoStream +from VideoStream import VideoStream +import AnnotationParser +from AnnotationParser import AnnotationParser +import ImageServer +from ImageServer import ImageServer class CameraCapture(object): - def __IsInt(self, string): - try: + def __IsInt(self,string): + try: int(string) return True except ValueError: @@ -36,26 +36,26 @@ def __IsInt(self, string): def __init__( self, videoPath, - imageProcessingEndpoint="", - imageProcessingParams="", - showVideo=False, - verbose=False, - loopVideo=True, - convertToGray=False, - resizeWidth=0, - resizeHeight=0, - annotate=False, - sendToHubCallback=None): + imageProcessingEndpoint = "", + imageProcessingParams = "", + showVideo = False, + verbose = False, + loopVideo = True, + convertToGray = False, + resizeWidth = 0, + resizeHeight = 0, + annotate = False, + sendToHubCallback = None): self.videoPath = videoPath if self.__IsInt(videoPath): - # case of a usb camera (usually mounted at /dev/video* where * is an int) + #case of a usb camera (usually mounted at /dev/video* where * is an int) self.isWebcam = True else: - # case of a video file + #case of a video file self.isWebcam = False self.imageProcessingEndpoint = imageProcessingEndpoint if imageProcessingParams == "": - self.imageProcessingParams = "" + self.imageProcessingParams = "" else: self.imageProcessingParams = json.loads(imageProcessingParams) self.showVideo = showVideo @@ -64,34 +64,30 @@ def __init__( self.convertToGray = convertToGray self.resizeWidth = resizeWidth self.resizeHeight = resizeHeight - self.annotate = (self.imageProcessingEndpoint != - "") and self.showVideo & annotate + self.annotate = (self.imageProcessingEndpoint != "") and self.showVideo & annotate self.nbOfPreprocessingSteps = 0 self.autoRotate = False self.sendToHubCallback = sendToHubCallback self.vs = None if self.convertToGray: - self.nbOfPreprocessingSteps += 1 + self.nbOfPreprocessingSteps +=1 if self.resizeWidth != 0 or self.resizeHeight != 0: - self.nbOfPreprocessingSteps += 1 + self.nbOfPreprocessingSteps +=1 if self.verbose: print("Initialising the camera capture with the following parameters: ") print(" - Video path: " + self.videoPath) - print(" - Image processing endpoint: " + - self.imageProcessingEndpoint) - print(" - Image processing params: " + - json.dumps(self.imageProcessingParams)) + print(" - Image processing endpoint: " + self.imageProcessingEndpoint) + print(" - Image processing params: " + json.dumps(self.imageProcessingParams)) print(" - Show video: " + str(self.showVideo)) print(" - Loop video: " + str(self.loopVideo)) print(" - Convert to gray: " + str(self.convertToGray)) print(" - Resize width: " + str(self.resizeWidth)) print(" - Resize height: " + str(self.resizeHeight)) print(" - Annotate: " + str(self.annotate)) - print(" - Send processing results to hub: " + - str(self.sendToHubCallback is not None)) + print(" - Send processing results to hub: " + str(self.sendToHubCallback is not None)) print() - + self.displayFrame = None if self.showVideo: self.imageServer = ImageServer(5012, self) @@ -99,30 +95,25 @@ def __init__( def __annotate(self, frame, response): AnnotationParserInstance = AnnotationParser() - # TODO: Make the choice of the service configurable - listOfRectanglesToDisplay = AnnotationParserInstance.getCV2RectanglesFromProcessingService1( - response) + #TODO: Make the choice of the service configurable + listOfRectanglesToDisplay = AnnotationParserInstance.getCV2RectanglesFromProcessingService1(response) for rectangle in listOfRectanglesToDisplay: - cv2.rectangle(frame, (rectangle(0), rectangle(1)), - (rectangle(2), rectangle(3)), (0, 0, 255), 4) + cv2.rectangle(frame, (rectangle(0), rectangle(1)), (rectangle(2), rectangle(3)), (0,0,255),4) return def __sendFrameForProcessing(self, frame): headers = {'Content-Type': 'application/octet-stream'} try: - response = requests.post( - self.imageProcessingEndpoint, headers=headers, params=self.imageProcessingParams, data=frame) + response = requests.post(self.imageProcessingEndpoint, headers = headers, params = self.imageProcessingParams, data = frame) except Exception as e: print('__sendFrameForProcessing Excpetion -' + str(e)) return "[]" if self.verbose: try: - print("Response from external processing service: (" + - str(response.status_code) + ") " + json.dumps(response.json())) + print("Response from external processing service: (" + str(response.status_code) + ") " + json.dumps(response.json())) except Exception: - print("Response from external processing service (status code): " + - str(response.status_code)) + print("Response from external processing service (status code): " + str(response.status_code)) return json.dumps(response.json()) def __displayTimeDifferenceInMs(self, endTime, startTime): @@ -130,13 +121,12 @@ def __displayTimeDifferenceInMs(self, endTime, startTime): def __enter__(self): if self.isWebcam: - # The VideoStream class always gives us the latest frame from the webcam. It uses another thread to read the frames. + #The VideoStream class always gives us the latest frame from the webcam. It uses another thread to read the frames. self.vs = VideoStream(int(self.videoPath)).start() - # needed to load at least one frame into the VideoStream class - time.sleep(1.0) + time.sleep(1.0)#needed to load at least one frame into the VideoStream class #self.capture = cv2.VideoCapture(int(self.videoPath)) else: - # In the case of a video file, we want to analyze all the frames of the video thus are not using VideoStream class + #In the case of a video file, we want to analyze all the frames of the video thus are not using VideoStream class self.capture = cv2.VideoCapture(self.videoPath) return self @@ -152,7 +142,7 @@ def start(self): if self.verbose: startCapture = time.time() - frameCounter += 1 + frameCounter +=1 if self.isWebcam: frame = self.vs.read() else: @@ -161,128 +151,107 @@ def start(self): if self.capture.get(cv2.CAP_PROP_FRAME_WIDTH) < self.capture.get(cv2.CAP_PROP_FRAME_HEIGHT): self.autoRotate = True if self.autoRotate: - # The counterclockwise is random...It coudl well be clockwise. Is there a way to auto detect it? - frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE) + frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE) #The counterclockwise is random...It coudl well be clockwise. Is there a way to auto detect it? if self.verbose: if frameCounter == 1: if not self.isWebcam: - print("Original frame size: " + str(int(self.capture.get(cv2.CAP_PROP_FRAME_WIDTH)) - ) + "x" + str(int(self.capture.get(cv2.CAP_PROP_FRAME_HEIGHT)))) - print("Frame rate (FPS): " + - str(int(self.capture.get(cv2.CAP_PROP_FPS)))) + print("Original frame size: " + str(int(self.capture.get(cv2.CAP_PROP_FRAME_WIDTH))) + "x" + str(int(self.capture.get(cv2.CAP_PROP_FRAME_HEIGHT)))) + print("Frame rate (FPS): " + str(int(self.capture.get(cv2.CAP_PROP_FPS)))) print("Frame number: " + str(frameCounter)) - print("Time to capture (+ straighten up) a frame: " + - self.__displayTimeDifferenceInMs(time.time(), startCapture)) + print("Time to capture (+ straighten up) a frame: " + self.__displayTimeDifferenceInMs(time.time(), startCapture)) startPreProcessing = time.time() - - # Loop video - if not self.isWebcam: + + #Loop video + if not self.isWebcam: if frameCounter == self.capture.get(cv2.CAP_PROP_FRAME_COUNT): - if self.loopVideo: + if self.loopVideo: frameCounter = 0 self.capture.set(cv2.CAP_PROP_POS_FRAMES, 0) else: break - # Pre-process locally + #Pre-process locally if self.nbOfPreprocessingSteps == 1 and self.convertToGray: preprocessedFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - + if self.nbOfPreprocessingSteps == 1 and (self.resizeWidth != 0 or self.resizeHeight != 0): - preprocessedFrame = cv2.resize( - frame, (self.resizeWidth, self.resizeHeight)) + preprocessedFrame = cv2.resize(frame, (self.resizeWidth, self.resizeHeight)) if self.nbOfPreprocessingSteps > 1: preprocessedFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - preprocessedFrame = cv2.resize( - preprocessedFrame, (self.resizeWidth, self.resizeHeight)) - + preprocessedFrame = cv2.resize(preprocessedFrame, (self.resizeWidth,self.resizeHeight)) + if self.verbose: - print("Time to pre-process a frame: " + - self.__displayTimeDifferenceInMs(time.time(), startPreProcessing)) + print("Time to pre-process a frame: " + self.__displayTimeDifferenceInMs(time.time(), startPreProcessing)) startEncodingForProcessing = time.time() - # Process externally + #Process externally if self.imageProcessingEndpoint != "": - # Encode frame to send over HTTP + #Encode frame to send over HTTP if self.nbOfPreprocessingSteps == 0: encodedFrame = cv2.imencode(".jpg", frame)[1].tostring() else: - encodedFrame = cv2.imencode(".jpg", preprocessedFrame)[ - 1].tostring() + encodedFrame = cv2.imencode(".jpg", preprocessedFrame)[1].tostring() if self.verbose: - print("Time to encode a frame for processing: " + - self.__displayTimeDifferenceInMs(time.time(), startEncodingForProcessing)) + print("Time to encode a frame for processing: " + self.__displayTimeDifferenceInMs(time.time(), startEncodingForProcessing)) startProcessingExternally = time.time() - # Send over HTTP for processing + #Send over HTTP for processing response = self.__sendFrameForProcessing(encodedFrame) if self.verbose: - print("Time to process frame externally: " + - self.__displayTimeDifferenceInMs(time.time(), startProcessingExternally)) + print("Time to process frame externally: " + self.__displayTimeDifferenceInMs(time.time(), startProcessingExternally)) startSendingToEdgeHub = time.time() - # forwarding outcome of external processing to the EdgeHub + #forwarding outcome of external processing to the EdgeHub if response != "[]" and self.sendToHubCallback is not None: self.sendToHubCallback(response) if self.verbose: - print("Time to message from processing service to edgeHub: " + - self.__displayTimeDifferenceInMs(time.time(), startSendingToEdgeHub)) + print("Time to message from processing service to edgeHub: " + self.__displayTimeDifferenceInMs(time.time(), startSendingToEdgeHub)) startDisplaying = time.time() - # Display frames + #Display frames if self.showVideo: try: if self.nbOfPreprocessingSteps == 0: if self.verbose and (perfForOneFrameInMs is not None): - cv2.putText(frame, "FPS " + str(round(1000/perfForOneFrameInMs, 2)), - (10, 35), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 255), 2) + cv2.putText(frame, "FPS " + str(round(1000/perfForOneFrameInMs, 2)),(10, 35),cv2.FONT_HERSHEY_SIMPLEX,1.0,(0,0,255), 2) if self.annotate: - # TODO: fix bug with annotate function + #TODO: fix bug with annotate function self.__annotate(frame, response) - self.displayFrame = cv2.imencode( - '.jpg', frame)[1].tobytes() + self.displayFrame = cv2.imencode('.jpg', frame)[1].tobytes() else: if self.verbose and (perfForOneFrameInMs is not None): - cv2.putText(preprocessedFrame, "FPS " + str(round(1000/perfForOneFrameInMs, 2)), - (10, 35), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 255), 2) + cv2.putText(preprocessedFrame, "FPS " + str(round(1000/perfForOneFrameInMs, 2)),(10, 35),cv2.FONT_HERSHEY_SIMPLEX,1.0,(0,0,255), 2) if self.annotate: - # TODO: fix bug with annotate function + #TODO: fix bug with annotate function self.__annotate(preprocessedFrame, response) - self.displayFrame = cv2.imencode( - '.jpg', preprocessedFrame)[1].tobytes() + self.displayFrame = cv2.imencode('.jpg', preprocessedFrame)[1].tobytes() except Exception as e: - print("Could not display the video to a web browser.") + print("Could not display the video to a web browser.") print('Excpetion -' + str(e)) if self.verbose: if 'startDisplaying' in locals(): - print("Time to display frame: " + - self.__displayTimeDifferenceInMs(time.time(), startDisplaying)) + print("Time to display frame: " + self.__displayTimeDifferenceInMs(time.time(), startDisplaying)) elif 'startSendingToEdgeHub' in locals(): - print("Time to display frame: " + - self.__displayTimeDifferenceInMs(time.time(), startSendingToEdgeHub)) + print("Time to display frame: " + self.__displayTimeDifferenceInMs(time.time(), startSendingToEdgeHub)) else: - print("Time to display frame: " + self.__displayTimeDifferenceInMs( - time.time(), startEncodingForProcessing)) + print("Time to display frame: " + self.__displayTimeDifferenceInMs(time.time(), startEncodingForProcessing)) perfForOneFrameInMs = int((time.time()-startOverall) * 1000) if not self.isWebcam: - waitTimeBetweenFrames = max( - int(1000 / self.capture.get(cv2.CAP_PROP_FPS))-perfForOneFrameInMs, 1) - print("Wait time between frames :" + - str(waitTimeBetweenFrames)) + waitTimeBetweenFrames = max(int(1000 / self.capture.get(cv2.CAP_PROP_FPS))-perfForOneFrameInMs, 1) + print("Wait time between frames :" + str(waitTimeBetweenFrames)) if cv2.waitKey(waitTimeBetweenFrames) & 0xFF == ord('q'): break if self.verbose: perfForOneFrameInMs = int((time.time()-startOverall) * 1000) - print("Total time for one frame: " + - self.__displayTimeDifferenceInMs(time.time(), startOverall)) + print("Total time for one frame: " + self.__displayTimeDifferenceInMs(time.time(), startOverall)) def __exit__(self, exception_type, exception_value, traceback): if not self.isWebcam: self.capture.release() if self.showVideo: self.imageServer.close() - cv2.destroyAllWindows() + cv2.destroyAllWindows() \ No newline at end of file diff --git a/modules/CameraCapture/module.json b/modules/CameraCapture/module.json index 932c084..591e895 100644 --- a/modules/CameraCapture/module.json +++ b/modules/CameraCapture/module.json @@ -4,7 +4,7 @@ "image": { "repository": "$CONTAINER_REGISTRY_ADDRESS/cameracapture", "tag": { - "version": "0.2.8", + "version": "0.2.11", "platforms": { "amd64": "./amd64.Dockerfile", "arm32v7": "./arm32v7.Dockerfile", diff --git a/modules/ImageClassifierService/amd64.Dockerfile b/modules/ImageClassifierService/amd64.Dockerfile index 6db8259..64ea84c 100644 --- a/modules/ImageClassifierService/amd64.Dockerfile +++ b/modules/ImageClassifierService/amd64.Dockerfile @@ -1,13 +1,9 @@ -FROM tensorflow/tensorflow:latest-py3 +FROM python:3.7-slim -RUN echo "BUILD MODULE: ImageClassifierService" +RUN pip install -U pip +RUN pip install numpy==1.17.3 tensorflow==2.0.0 flask pillow -COPY /build/amd64-requirements.txt amd64-requirements.txt - -# Install Python packages -RUN pip install -r amd64-requirements.txt - -ADD app /app +COPY app /app # Expose the port EXPOSE 80 @@ -16,4 +12,4 @@ EXPOSE 80 WORKDIR /app # Run the flask server for the endpoints -CMD python app.py +CMD python -u app.py diff --git a/modules/ImageClassifierService/app/app.py b/modules/ImageClassifierService/app/app.py index 423453e..def1cf9 100644 --- a/modules/ImageClassifierService/app/app.py +++ b/modules/ImageClassifierService/app/app.py @@ -4,12 +4,10 @@ import io # Imports for the REST API -from flask import Flask, request +from flask import Flask, request, jsonify # Imports for image procesing from PIL import Image -#import scipy -#from scipy import misc # Imports for prediction from predict import initialize, predict_image, predict_url @@ -17,7 +15,7 @@ app = Flask(__name__) # 4MB Max image size limit -app.config['MAX_CONTENT_LENGTH'] = 4 * 1024 * 1024 +app.config['MAX_CONTENT_LENGTH'] = 4 * 1024 * 1024 # Default route just shows simple text @app.route('/') @@ -25,21 +23,28 @@ def index(): return 'CustomVision.ai model host harness' # Like the CustomVision.ai Prediction service /image route handles either -# - octet-stream image file +# - octet-stream image file # - a multipart/form-data with files in the imageData parameter @app.route('/image', methods=['POST']) -def predict_image_handler(): +@app.route('//image', methods=['POST']) +@app.route('//image/nostore', methods=['POST']) +@app.route('//classify/iterations//image', methods=['POST']) +@app.route('//classify/iterations//image/nostore', methods=['POST']) +@app.route('//detect/iterations//image', methods=['POST']) +@app.route('//detect/iterations//image/nostore', methods=['POST']) +def predict_image_handler(project=None, publishedName=None): try: imageData = None if ('imageData' in request.files): imageData = request.files['imageData'] + elif ('imageData' in request.form): + imageData = request.form['imageData'] else: imageData = io.BytesIO(request.get_data()) - #img = scipy.misc.imread(imageData) img = Image.open(imageData) results = predict_image(img) - return json.dumps(results) + return jsonify(results) except Exception as e: print('EXCEPTION:', str(e)) return 'Error processing image', 500 @@ -47,21 +52,27 @@ def predict_image_handler(): # Like the CustomVision.ai Prediction service /url route handles url's # in the body of hte request of the form: -# { 'Url': ''} +# { 'Url': ''} @app.route('/url', methods=['POST']) -def predict_url_handler(): +@app.route('//url', methods=['POST']) +@app.route('//url/nostore', methods=['POST']) +@app.route('//classify/iterations//url', methods=['POST']) +@app.route('//classify/iterations//url/nostore', methods=['POST']) +@app.route('//detect/iterations//url', methods=['POST']) +@app.route('//detect/iterations//url/nostore', methods=['POST']) +def predict_url_handler(project=None, publishedName=None): try: - image_url = json.loads(request.get_data())['Url'] + image_url = json.loads(request.get_data().decode('utf-8'))['url'] results = predict_url(image_url) - return json.dumps(results) + return jsonify(results) except Exception as e: print('EXCEPTION:', str(e)) return 'Error processing image' - if __name__ == '__main__': # Load and intialize the model initialize() # Run the server app.run(host='0.0.0.0', port=80) + diff --git a/modules/ImageClassifierService/app/predict.py b/modules/ImageClassifierService/app/predict.py index 70b6dff..1648d0a 100644 --- a/modules/ImageClassifierService/app/predict.py +++ b/modules/ImageClassifierService/app/predict.py @@ -1,121 +1,221 @@ from urllib.request import urlopen +from datetime import datetime -import tensorflow.compat.v1 as tf +import tensorflow as tf from PIL import Image import numpy as np -# import scipy -# from scipy import misc import sys -import os filename = 'model.pb' labels_filename = 'labels.txt' -mean_values_b_g_r = (0, 0, 0) +network_input_size = 0 -size = (256, 256) output_layer = 'loss:0' input_node = 'Placeholder:0' -graph_def = tf.GraphDef() +graph_def = tf.compat.v1.GraphDef() labels = [] - def initialize(): - print('Loading model...', end=''), - with tf.gfile.FastGFile(filename, 'rb') as f: + print('Loading model...',end=''), + with open(filename, 'rb') as f: graph_def.ParseFromString(f.read()) tf.import_graph_def(graph_def, name='') + + # Retrieving 'network_input_size' from shape of 'input_node' + with tf.compat.v1.Session() as sess: + input_tensor_shape = sess.graph.get_tensor_by_name(input_node).shape.as_list() + + assert len(input_tensor_shape) == 4 + assert input_tensor_shape[1] == input_tensor_shape[2] + + global network_input_size + network_input_size = input_tensor_shape[1] + print('Success!') print('Loading labels...', end='') with open(labels_filename, 'rt') as lf: - for l in lf: - l = l[:-1] - labels.append(l) + global labels + labels = [l.strip() for l in lf.readlines()] print(len(labels), 'found. Success!') - -def crop_center(img, cropx, cropy): - y, x, z = img.shape - startx = x//2-(cropx//2) - starty = y//2-(cropy//2) - print('crop_center: ', x, 'x', y, 'to', cropx, 'x', cropy) +def log_msg(msg): + print("{}: {}".format(datetime.now(),msg)) + +def extract_bilinear_pixel(img, x, y, ratio, xOrigin, yOrigin): + xDelta = (x + 0.5) * ratio - 0.5 + x0 = int(xDelta) + xDelta -= x0 + x0 += xOrigin + if x0 < 0: + x0 = 0; + x1 = 0; + xDelta = 0.0; + elif x0 >= img.shape[1]-1: + x0 = img.shape[1]-1; + x1 = img.shape[1]-1; + xDelta = 0.0; + else: + x1 = x0 + 1; + + yDelta = (y + 0.5) * ratio - 0.5 + y0 = int(yDelta) + yDelta -= y0 + y0 += yOrigin + if y0 < 0: + y0 = 0; + y1 = 0; + yDelta = 0.0; + elif y0 >= img.shape[0]-1: + y0 = img.shape[0]-1; + y1 = img.shape[0]-1; + yDelta = 0.0; + else: + y1 = y0 + 1; + + #Get pixels in four corners + bl = img[y0, x0] + br = img[y0, x1] + tl = img[y1, x0] + tr = img[y1, x1] + #Calculate interpolation + b = xDelta * br + (1. - xDelta) * bl + t = xDelta * tr + (1. - xDelta) * tl + pixel = yDelta * t + (1. - yDelta) * b + return pixel + +def extract_and_resize(img, targetSize): + determinant = img.shape[1] * targetSize[0] - img.shape[0] * targetSize[1] + if determinant < 0: + ratio = float(img.shape[1]) / float(targetSize[1]) + xOrigin = 0 + yOrigin = int(0.5 * (img.shape[0] - ratio * targetSize[0])) + elif determinant > 0: + ratio = float(img.shape[0]) / float(targetSize[0]) + xOrigin = int(0.5 * (img.shape[1] - ratio * targetSize[1])) + yOrigin = 0 + else: + ratio = float(img.shape[0]) / float(targetSize[0]) + xOrigin = 0 + yOrigin = 0 + resize_image = np.empty((targetSize[0], targetSize[1], img.shape[2]), dtype=np.float32) + for y in range(targetSize[0]): + for x in range(targetSize[1]): + resize_image[y, x] = extract_bilinear_pixel(img, x, y, ratio, xOrigin, yOrigin) + return resize_image + +def extract_and_resize_to_256_square(image): + h, w = image.shape[:2] + log_msg("crop_center: " + str(w) + "x" + str(h) +" and resize to " + str(256) + "x" + str(256)) + return extract_and_resize(image, (256, 256)) + +def crop_center(img,cropx,cropy): + h, w = img.shape[:2] + startx = max(0, w//2-(cropx//2)) + starty = max(0, h//2-(cropy//2)) + log_msg("crop_center: " + str(w) + "x" + str(h) +" to " + str(cropx) + "x" + str(cropy)) return img[starty:starty+cropy, startx:startx+cropx] +def resize_down_to_1600_max_dim(image): + w,h = image.size + if h < 1600 and w < 1600: + return image + + new_size = (1600 * w // h, 1600) if (h > w) else (1600, 1600 * h // w) + log_msg("resize: " + str(w) + "x" + str(h) + " to " + str(new_size[0]) + "x" + str(new_size[1])) + if max(new_size) / max(image.size) >= 0.5: + method = Image.BILINEAR + else: + method = Image.BICUBIC + return image.resize(new_size, method) def predict_url(imageUrl): - print('Predicting from url: ', imageUrl) + log_msg("Predicting from url: " +imageUrl) with urlopen(imageUrl) as testImage: - # image = scipy.misc.imread(testImage) image = Image.open(testImage) return predict_image(image) - +def convert_to_nparray(image): + # RGB -> BGR + log_msg("Convert to numpy array") + image = np.array(image) + return image[:, :, (2,1,0)] + +def update_orientation(image): + exif_orientation_tag = 0x0112 + if hasattr(image, '_getexif'): + exif = image._getexif() + if exif != None and exif_orientation_tag in exif: + orientation = exif.get(exif_orientation_tag, 1) + log_msg('Image has EXIF Orientation: ' + str(orientation)) + # orientation is 1 based, shift to zero based and flip/transpose based on 0-based values + orientation -= 1 + if orientation >= 4: + image = image.transpose(Image.TRANSPOSE) + if orientation == 2 or orientation == 3 or orientation == 6 or orientation == 7: + image = image.transpose(Image.FLIP_TOP_BOTTOM) + if orientation == 1 or orientation == 2 or orientation == 5 or orientation == 6: + image = image.transpose(Image.FLIP_LEFT_RIGHT) + return image + def predict_image(image): - print('Predicting image') - tf.reset_default_graph() - tf.import_graph_def(graph_def, name='') - - with tf.Session() as sess: - prob_tensor = sess.graph.get_tensor_by_name(output_layer) - - input_tensor_shape = sess.graph.get_tensor_by_name( - 'Placeholder:0').shape.as_list() - network_input_size = input_tensor_shape[1] - - # w = image.shape[0] - # h = image.shape[1] - w, h = image.size - print('Image size', w, 'x', h) - - # scaling - if w > h: - new_size = (int((float(size[1]) / h) * w), size[1], 3) - else: - new_size = (size[0], int((float(size[0]) / w) * h), 3) - - # resize - if not (new_size[0] == w and new_size[0] == h): - print('Resizing to', new_size[0], 'x', new_size[1]) - #augmented_image = scipy.misc.imresize(image, new_size) - augmented_image = np.asarray( - image.resize((new_size[0], new_size[1]))) - else: - augmented_image = np.asarray(image) - - # crop center - try: - augmented_image = crop_center( - augmented_image, network_input_size, network_input_size) - except: - return 'error: crop_center' - - augmented_image = augmented_image.astype(float) - - # RGB -> BGR - red, green, blue = tf.split( - axis=2, num_or_size_splits=3, value=augmented_image) - - image_normalized = tf.concat(axis=2, values=[ - blue - mean_values_b_g_r[0], - green - mean_values_b_g_r[1], - red - mean_values_b_g_r[2], - ]) - - image_normalized = image_normalized.eval() - image_normalized = np.expand_dims(image_normalized, axis=0) - - predictions, = sess.run(prob_tensor, {input_node: image_normalized}) - - result = [] - idx = 0 - for p in predictions: - truncated_probablity = np.float64(round(p, 8)) - if (truncated_probablity > 1e-8): - result.append( - {'Tag': labels[idx], 'Probability': truncated_probablity}) - idx += 1 - print('Results: ', str(result)) - return result + + log_msg('Predicting image') + try: + if image.mode != "RGB": + log_msg("Converting to RGB") + image = image.convert("RGB") + + w,h = image.size + log_msg("Image size: " + str(w) + "x" + str(h)) + + # Update orientation based on EXIF tags + image = update_orientation(image) + + # If the image has either w or h greater than 1600 we resize it down respecting + # aspect ratio such that the largest dimention is 1600 + image = resize_down_to_1600_max_dim(image) + + # Convert image to numpy array + image = convert_to_nparray(image) + + # Crop the center square and resize that square down to 256x256 + resized_image = extract_and_resize_to_256_square(image) + + # Crop the center for the specified network_input_Size + cropped_image = crop_center(resized_image, network_input_size, network_input_size) + + tf.compat.v1.reset_default_graph() + tf.import_graph_def(graph_def, name='') + + with tf.compat.v1.Session() as sess: + prob_tensor = sess.graph.get_tensor_by_name(output_layer) + predictions, = sess.run(prob_tensor, {input_node: [cropped_image] }) + + result = [] + for p, label in zip(predictions, labels): + truncated_probablity = np.float64(round(p,8)) + if truncated_probablity > 1e-8: + result.append({ + 'tagName': label, + 'probability': truncated_probablity, + 'tagId': '', + 'boundingBox': None }) + + response = { + 'id': '', + 'project': '', + 'iteration': '', + 'created': datetime.utcnow().isoformat(), + 'predictions': result + } + + log_msg("Results: " + str(response)) + return response + + except Exception as e: + log_msg(str(e)) + return 'Error: Could not preprocess image for prediction. ' + str(e) diff --git a/modules/ImageClassifierService/arm32v7.Dockerfile b/modules/ImageClassifierService/arm32v7.Dockerfile index f64fc6c..c43c285 100644 --- a/modules/ImageClassifierService/arm32v7.Dockerfile +++ b/modules/ImageClassifierService/arm32v7.Dockerfile @@ -1,38 +1,13 @@ -FROM balenalib/raspberrypi3:stretch -# The balena base image for building apps on Raspberry Pi 3. -# Raspbian Stretch required for piwheels support. https://downloads.raspberrypi.org/raspbian/images/raspbian-2019-04-09/ - -RUN echo "BUILD MODULE: ImageClassifierService" +FROM balenalib/raspberrypi3-debian-python:3.7 RUN [ "cross-build-start" ] -# Install dependencies -RUN install_packages \ - python3 \ - python3-pip \ - python3-dev \ - build-essential \ - libopenjp2-7-dev \ - libtiff5-dev \ - zlib1g-dev \ - libjpeg-dev \ - libatlas-base-dev \ - wget - -# Install Python packages -COPY /build/arm32v7-requirements.txt ./ -RUN pip3 install --upgrade pip -RUN pip3 install --upgrade setuptools -RUN pip3 install --index-url=https://www.piwheels.org/simple -r arm32v7-requirements.txt +RUN apt update && apt install -y libjpeg62-turbo libopenjp2-7 libtiff5 libatlas-base-dev +RUN pip install absl-py six protobuf wrapt gast astor termcolor keras_applications keras_preprocessing --no-deps +RUN pip install numpy==1.16 tensorflow==1.13.1 --extra-index-url 'https://www.piwheels.org/simple' --no-deps +RUN pip install flask pillow --index-url 'https://www.piwheels.org/simple' -# Cleanup -RUN rm -rf /var/lib/apt/lists/* \ - && apt-get -y autoremove - -RUN [ "cross-build-end" ] - -# Add the application -ADD app /app +COPY app /app # Expose the port EXPOSE 80 @@ -40,5 +15,7 @@ EXPOSE 80 # Set the working directory WORKDIR /app +RUN [ "cross-build-end" ] + # Run the flask server for the endpoints -CMD ["python3","app.py"] +CMD python -u app.py \ No newline at end of file diff --git a/modules/ImageClassifierService/build/amd64-requirements.txt b/modules/ImageClassifierService/build/amd64-requirements.txt deleted file mode 100644 index 9f82ee2..0000000 --- a/modules/ImageClassifierService/build/amd64-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pillow -numpy -flask diff --git a/modules/ImageClassifierService/build/arm32v7-requirements.txt b/modules/ImageClassifierService/build/arm32v7-requirements.txt deleted file mode 100644 index 15fbf9e..0000000 --- a/modules/ImageClassifierService/build/arm32v7-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pillow -numpy -flask -tensorflow==1.12.2 diff --git a/modules/ImageClassifierService/module.json b/modules/ImageClassifierService/module.json index c05388a..2718f6c 100644 --- a/modules/ImageClassifierService/module.json +++ b/modules/ImageClassifierService/module.json @@ -4,7 +4,7 @@ "image": { "repository": "$CONTAINER_REGISTRY_ADDRESS/imageclassifierservice", "tag": { - "version": "0.2.5", + "version": "0.2.6", "platforms": { "amd64": "./amd64.Dockerfile", "arm32v7": "./arm32v7.Dockerfile" diff --git a/modules/SenseHatDisplay/app/MessageParser.py b/modules/SenseHatDisplay/app/MessageParser.py index eca2672..a59eb33 100644 --- a/modules/SenseHatDisplay/app/MessageParser.py +++ b/modules/SenseHatDisplay/app/MessageParser.py @@ -1,10 +1,10 @@ class MessageParser: # Returns the highest probablity tag in the json object (takes the output as json.loads as input) - def highestProbabilityTagMeetingThreshold(self, allTagsAndProbability, threshold): + def highestProbabilityTagMeetingThreshold(self, message, threshold): highestProbabilityTag = 'none' highestProbability = 0 - for item in allTagsAndProbability: - if item['Probability'] > highestProbability and item['Probability'] > threshold: - highestProbability = item['Probability'] - highestProbabilityTag = item['Tag'] + for prediction in message['predictions']: + if prediction['probability'] > highestProbability and prediction['probability'] > threshold: + highestProbability = prediction['probability'] + highestProbabilityTag = prediction['tagName'] return highestProbabilityTag diff --git a/modules/SenseHatDisplay/app/__init__.py b/modules/SenseHatDisplay/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/SenseHatDisplay/app/main.py b/modules/SenseHatDisplay/app/main.py index d5563ad..f153536 100644 --- a/modules/SenseHatDisplay/app/main.py +++ b/modules/SenseHatDisplay/app/main.py @@ -47,7 +47,7 @@ def __init__(self): self.client_protocol = protocol self.client = IoTHubModuleClient() self.client.create_from_environment(protocol) - self.client.set_option("logtrace", 1) # enables MQTT logging + self.client.set_option("logtrace", 0) # enables MQTT logging self.client.set_option("messageTimeout", 10000) # sets the callback when a message arrives on "input1" queue. Messages sent to diff --git a/modules/SenseHatDisplay/module.json b/modules/SenseHatDisplay/module.json index 53116e7..98f249c 100644 --- a/modules/SenseHatDisplay/module.json +++ b/modules/SenseHatDisplay/module.json @@ -4,7 +4,7 @@ "image": { "repository": "$CONTAINER_REGISTRY_ADDRESS/sensehatdisplay", "tag": { - "version": "0.2.12", + "version": "0.2.13", "platforms": { "arm32v7": "./arm32v7.Dockerfile", "test-arm32v7": "./test/test-arm32v7.Dockerfile" diff --git a/modules/SenseHatDisplay/test/UnitTests.py b/modules/SenseHatDisplay/test/UnitTests.py index 2708391..ec5f2da 100644 --- a/modules/SenseHatDisplay/test/UnitTests.py +++ b/modules/SenseHatDisplay/test/UnitTests.py @@ -1,25 +1,24 @@ -import app.MessageParser import unittest import json import sys sys.path.insert(0, '../') - +import app.MessageParser class UnitTests(unittest.TestCase): def test_HighestProbabilityTagMeetingThreshold(self): MessageParser = app.MessageParser.MessageParser() message1 = json.loads( - "[{\"Tag\": \"banana\",\"Probability\": 0.4}, {\"Tag\": \"apple\",\"Probability\": 0.3}]") + "{\"iteration\": \"\",\"id\": \"\",\"predictions\": [{\"probability\": 0.3,\"tagName\": \"Apple\",\"tagId\": \"\",\"boundingBox\": null},{\"probability\": 0.4,\"tagName\": \"Banana\",\"tagId\": \"\",\"boundingBox\": null}],\"project\": \"\",\"created\": \"2019-12-10T04:37:49.657555\"}") self.assertEqual( MessageParser.highestProbabilityTagMeetingThreshold(message1, 0.5), 'none') message2 = json.loads( - "[{\"Tag\": \"banana\",\"Probability\": 0.4}, {\"Tag\": \"apple\",\"Probability\": 0.5}]") + "{\"iteration\": \"\",\"id\": \"\",\"predictions\": [{\"probability\": 0.5,\"tagName\": \"Apple\",\"tagId\": \"\",\"boundingBox\": null},{\"probability\": 0.4,\"tagName\": \"Banana\",\"tagId\": \"\",\"boundingBox\": null}],\"project\": \"\",\"created\": \"2019-12-10T04:37:49.657555\"}") self.assertEqual(MessageParser.highestProbabilityTagMeetingThreshold( - message2, 0.3), 'apple') + message2, 0.3), 'Apple') message3 = json.loads( - "[{\"Probability\": 0.038001421838998795, \"Tag\": \"apple\"}, {\"Probability\": 0.38567957282066345, \"Tag\": \"banana\"}]") + "{\"iteration\": \"\",\"id\": \"\",\"predictions\": [{\"probability\": 0.038001421838998795,\"tagName\": \"Apple\",\"tagId\": \"\",\"boundingBox\": null},{\"probability\": 0.38567957282066345,\"tagName\": \"Banana\",\"tagId\": \"\",\"boundingBox\": null}],\"project\": \"\",\"created\": \"2019-12-10T04:37:49.657555\"}") self.assertEqual(MessageParser.highestProbabilityTagMeetingThreshold( - message3, 0.3), 'banana') + message3, 0.3), 'Banana') if __name__ == '__main__':