Merge pull request #44 from shawncal/shawncal/python3-update

Python3.5 Update, Build speed improvements
Azure-Samples · Apr 12, 2019 · 08dfd56 · 08dfd56
2 parents 98dd952 + fd13cc7
commit 08dfd56
Show file tree

Hide file tree

Showing 17 changed files with 228 additions and 205 deletions.
diff --git a/modules/CameraCapture/amd64.Dockerfile b/modules/CameraCapture/amd64.Dockerfile
@@ -1,24 +1,31 @@
 FROM ubuntu:xenial
 
+RUN echo "BUILD MODULE: CameraCapture"
+
 WORKDIR /app
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends libcurl4-openssl-dev python-pip libboost-python-dev && \
-    rm -rf /var/lib/apt/lists/* 
-
+    apt-get install -y --no-install-recommends \
+        python3 \
+        python3-pip \
+        python3-dev \
+        libcurl4-openssl-dev \
+        libboost-python-dev \
+        libgtk2.0-dev
+
+# Install Python packages
 COPY /build/amd64-requirements.txt ./
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools && pip install -r amd64-requirements.txt
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends libgtk2.0-dev && \
-    rm -rf /var/lib/apt/lists/*
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install -r amd64-requirements.txt
 
-RUN pip install trollius tornado
+# Cleanup
+RUN rm -rf /var/lib/apt/lists/* \
+    && apt-get -y autoremove
 
 ADD /app/ .
 
 # Expose the port
 EXPOSE 5012
 
-ENTRYPOINT [ "python", "-u", "./main.py" ]
+ENTRYPOINT [ "python3", "-u", "./main.py" ]
diff --git a/modules/CameraCapture/app/CameraCapture.py b/modules/CameraCapture/app/CameraCapture.py
@@ -103,7 +103,12 @@ def __annotate(self, frame, response):
 
     def __sendFrameForProcessing(self, frame):
         headers = {'Content-Type': 'application/octet-stream'}
-        response = requests.post(self.imageProcessingEndpoint, headers = headers, params = self.imageProcessingParams, data = frame)
+        try:
+            response = requests.post(self.imageProcessingEndpoint, headers = headers, params = self.imageProcessingParams, data = frame)
+        except Exception as e:
+            print('__sendFrameForProcessing Excpetion -' + str(e))
+            return "[]"
+
         if self.verbose:
             try:
                 print("Response from external processing service: (" + str(response.status_code) + ") " + json.dumps(response.json()))
@@ -121,7 +126,7 @@ def __enter__(self):
             time.sleep(1.0)#needed to load at least one frame into the VideoStream class
             #self.capture = cv2.VideoCapture(int(self.videoPath))
         else:
-            #In the case of a video file, we want to analyze all the frames fo the video thus are not using VideoStream class
+            #In the case of a video file, we want to analyze all the frames of the video thus are not using VideoStream class
             self.capture = cv2.VideoCapture(self.videoPath)
         return self
 
@@ -198,7 +203,7 @@ def start(self):
                 if self.verbose:
                     print("Time to process frame externally: " + self.__displayTimeDifferenceInMs(time.time(), startProcessingExternally))
                     startSendingToEdgeHub = time.time()
-                
+
                 #forwarding outcome of external processing to the EdgeHub
                 if response != "[]" and self.sendToHubCallback is not None:
                     self.sendToHubCallback(response)

diff --git a/modules/CameraCapture/arm32v7.Dockerfile b/modules/CameraCapture/arm32v7.Dockerfile
@@ -1,60 +1,47 @@
-FROM resin/raspberrypi3-python:2.7
-#This image is base on the resin image for building Python apps on Raspberry Pi 3.
+FROM balenalib/raspberrypi3
+# The balena base image for building apps on Raspberry Pi 3.
 
-#Enforces cross-compilation through Quemu
-RUN [ "cross-build-start" ]
-
-#update list of packages available
-RUN apt-get update && apt-get install -y libboost-python1.55.0
+RUN echo "BUILD MODULE: CameraCapture"
 
-#Install python packages        
-COPY /build/arm32v7-requirements.txt ./
-RUN pip install --upgrade pip && pip install --upgrade setuptools && pip install -r arm32v7-requirements.txt
+# Enforces cross-compilation through Quemu
+RUN [ "cross-build-start" ]
 
-# Install build modules for openCV
-# Based on the work at https://github.com/mohaseeb/raspberrypi3-opencv-docker
-RUN sudo apt-get install -y --no-install-recommends \
-    # to build and install opencv
-    unzip \
-    build-essential cmake pkg-config \
-    # to work with image files
+# Update package index and install dependencies
+RUN install_packages \
+    python3 \
+    python3-pip \
+    python3-dev \
+    build-essential \
+    libopenjp2-7-dev \
+    zlib1g-dev \
+    libatlas-base-dev \
+    wget \
+    libboost-python1.62.0 \
+    curl \
+    libcurl4-openssl-dev
+
+# Required for OpenCV
+RUN install_packages \
+    # Hierarchical Data Format
+    libhdf5-dev libhdf5-serial-dev \
+    # for image files
     libjpeg-dev libtiff5-dev libjasper-dev libpng-dev \
-    # to work with video files
+    # for video files
     libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \
-    # to display GUI
-    libgtk2.0-dev pkg-config \
-    && sudo rm -rf /var/lib/apt/lists/* \
-    && sudo apt-get -y autoremove
+    # for gui
+    libqt4-test libqtgui4 libqtwebkit4 libgtk2.0-dev \
+    # high def image processing
+    libilmbase-dev libopenexr-dev
 
-RUN  OPENCV_VERSION=3.4.2 \
-  && WS_DIR=`pwd` \
-  && mkdir opencv \
-  && cd opencv \
-  # download OpenCV and opencv_contrib
-  && wget -O opencv.zip https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip \
-  && unzip opencv.zip \
-  && sudo rm -rf opencv.zip \
-  && OPENCV_SRC_DIR=`pwd`/opencv-$OPENCV_VERSION \
-  # build and install without gpu dependency
-  && cd $OPENCV_SRC_DIR \
-  && mkdir build && cd build \
-  && cmake -D CMAKE_BUILD_TYPE=RELEASE \
-    -D CMAKE_INSTALL_PREFIX=/usr/local \
-    -D ENABLE_FAST_MATH=1 \
-    -D CUDA_FAST_MATH=1 \
-    -D WITH_OPENCL=off -D WITH_OPENCL_SVM=off \
-    -D WITH_OPENCLAMDFFT=off \
-    -D WITH_OPENCLAMDBLAS=off \
-    -D OPENCV_EXTRA_MODULES_PATH=$OPENCV_CONTRIB_MODULES_SRC_DIR \
-    -D BUILD_opencv_gpu=off \
-    .. \
-  && make \
-  && sudo make install \
-  # cleanup
-  && cd $WS_DIR \
-  && sudo rm -rf opencv
+# Install Python packages
+COPY /build/arm32v7-requirements.txt ./
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --index-url=https://www.piwheels.org/simple -r arm32v7-requirements.txt
 
-RUN pip install trollius tornado
+# Cleanup
+RUN rm -rf /var/lib/apt/lists/* \
+    && apt-get -y autoremove
 
 RUN [ "cross-build-end" ]  
 
@@ -63,4 +50,4 @@ ADD /app/ .
 # Expose the port
 EXPOSE 5012
 
-ENTRYPOINT [ "python", "-u", "./main.py" ]
+ENTRYPOINT [ "python3", "-u", "./main.py" ]
diff --git a/modules/CameraCapture/build/amd64-requirements.txt b/modules/CameraCapture/build/amd64-requirements.txt
@@ -1,4 +1,6 @@
-azure-iothub-device-client==1.4.0
+azure-iothub-device-client
 numpy
-opencv-python
-requests
+opencv-contrib-python
+requests
+trollius
+tornado==4.5.3
diff --git a/modules/CameraCapture/build/arm32v7-requirements.txt b/modules/CameraCapture/build/arm32v7-requirements.txt
@@ -1,3 +1,6 @@
+azure-iothub-device-client
 numpy
+opencv-contrib-python
 requests
-azure-iothub-device-client==1.4.0
+trollius
+tornado==4.5.3
diff --git a/modules/CameraCapture/module.json b/modules/CameraCapture/module.json
@@ -4,7 +4,7 @@
     "image": {
         "repository": "$CONTAINER_REGISTRY_ADDRESS/cameracapture",
         "tag": {
-            "version": "0.2.7",
+            "version": "0.2.8",
             "platforms": {
                 "amd64": "./amd64.Dockerfile",
                 "arm32v7": "./arm32v7.Dockerfile",

diff --git a/modules/CameraCapture/test-amd64.Dockerfile b/modules/CameraCapture/test-amd64.Dockerfile
@@ -1,20 +1,27 @@
 FROM ubuntu:xenial
 
+RUN echo "BUILD MODULE: CameraCapture"
+
 WORKDIR /app
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends libcurl4-openssl-dev python-pip libboost-python-dev && \
-    rm -rf /var/lib/apt/lists/* 
-
+    apt-get install -y --no-install-recommends \
+        python3 \
+        python3-pip \
+        python3-dev \
+        libcurl4-openssl-dev \
+        libboost-python-dev \
+        libgtk2.0-dev
+
+# Install Python packages
 COPY /build/amd64-requirements.txt ./
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools && pip install -r amd64-requirements.txt
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends libgtk2.0-dev && \
-    rm -rf /var/lib/apt/lists/*
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools 
+RUN pip3 install -r amd64-requirements.txt
 
-RUN pip install trollius tornado
+# Cleanup
+RUN rm -rf /var/lib/apt/lists/* \
+    && apt-get -y autoremove
 
 ADD /app/ .
 ADD /test/ .
@@ -23,4 +30,4 @@ ADD /test/ .
 EXPOSE 5012
 
 #Manually run the main.py or test other functions
-ENTRYPOINT [ "python", "-u", "./main.py" ]
+ENTRYPOINT [ "python3", "-u", "./main.py" ]
diff --git a/modules/CameraCapture/test-arm32v7.Dockerfile b/modules/CameraCapture/test-arm32v7.Dockerfile
@@ -1,60 +1,47 @@
-FROM resin/raspberrypi3-python:2.7
-#This image is base on the resin image for building Python apps on Raspberry Pi 3.
+FROM balenalib/raspberrypi3
+# The balena base image for building apps on Raspberry Pi 3.
 
-#Enforces cross-compilation through Quemu
-RUN [ "cross-build-start" ]
-
-#update list of packages available
-RUN apt-get update && apt-get install -y libboost-python1.55.0
+RUN echo "BUILD MODULE: CameraCapture"
 
-#Install python packages        
-COPY /build/arm32v7-requirements.txt ./
-RUN pip install --upgrade pip && pip install --upgrade setuptools && pip install -r arm32v7-requirements.txt
+# Enforces cross-compilation through Quemu
+RUN [ "cross-build-start" ]
 
-# Install build modules for openCV
-# Based on the work at https://github.com/mohaseeb/raspberrypi3-opencv-docker
-RUN sudo apt-get install -y --no-install-recommends \
-    # to build and install opencv
-    unzip \
-    build-essential cmake pkg-config \
-    # to work with image files
+# Update package index and install dependencies
+RUN install_packages \
+    python3 \
+    python3-pip \
+    python3-dev \
+    build-essential \
+    libopenjp2-7-dev \
+    zlib1g-dev \
+    libatlas-base-dev \
+    wget \
+    libboost-python1.62.0 \
+    curl \
+    libcurl4-openssl-dev
+
+# Required for OpenCV
+RUN install_packages \
+    # Hierarchical Data Format
+    libhdf5-dev libhdf5-serial-dev \
+    # for image files
     libjpeg-dev libtiff5-dev libjasper-dev libpng-dev \
-    # to work with video files
+    # for video files
     libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \
-    # to display GUI
-    libgtk2.0-dev pkg-config \
-    && sudo rm -rf /var/lib/apt/lists/* \
-    && sudo apt-get -y autoremove
+    # for GUI
+    libqt4-test libqtgui4 libqtwebkit4 libgtk2.0-dev \
+    # high def image processing
+    libilmbase-dev libopenexr-dev
 
-RUN  OPENCV_VERSION=3.4.2 \
-  && WS_DIR=`pwd` \
-  && mkdir opencv \
-  && cd opencv \
-  # download OpenCV and opencv_contrib
-  && wget -O opencv.zip https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip \
-  && unzip opencv.zip \
-  && sudo rm -rf opencv.zip \
-  && OPENCV_SRC_DIR=`pwd`/opencv-$OPENCV_VERSION \
-  # build and install without gpu dependency
-  && cd $OPENCV_SRC_DIR \
-  && mkdir build && cd build \
-  && cmake -D CMAKE_BUILD_TYPE=RELEASE \
-    -D CMAKE_INSTALL_PREFIX=/usr/local \
-    -D ENABLE_FAST_MATH=1 \
-    -D CUDA_FAST_MATH=1 \
-    -D WITH_OPENCL=off -D WITH_OPENCL_SVM=off \
-    -D WITH_OPENCLAMDFFT=off \
-    -D WITH_OPENCLAMDBLAS=off \
-    -D OPENCV_EXTRA_MODULES_PATH=$OPENCV_CONTRIB_MODULES_SRC_DIR \
-    -D BUILD_opencv_gpu=off \
-    .. \
-  && make \
-  && sudo make install \
-  # cleanup
-  && cd $WS_DIR \
-  && sudo rm -rf opencv
+# Install Python packages
+COPY /build/arm32v7-requirements.txt ./
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --index-url=https://www.piwheels.org/simple -r arm32v7-requirements.txt
 
-RUN pip install trollius tornado
+# Cleanup
+RUN rm -rf /var/lib/apt/lists/* \
+    && apt-get -y autoremove
 
 RUN [ "cross-build-end" ]
 
@@ -64,4 +51,4 @@ ADD /test/ .
 # Expose the port
 EXPOSE 5012
 
-ENTRYPOINT [ "python", "-u", "./main.py" ]
+ENTRYPOINT [ "python3", "-u", "./main.py" ]
diff --git a/modules/ImageClassifierService/amd64.Dockerfile b/modules/ImageClassifierService/amd64.Dockerfile
@@ -1,7 +1,10 @@
 FROM tensorflow/tensorflow:latest-py3
 
+RUN echo "BUILD MODULE: ImageClassifierService"
+
 COPY /build/amd64-requirements.txt amd64-requirements.txt
 
+# Install Python packages
 RUN pip install -r amd64-requirements.txt
 
 ADD app /app