Merge pull request openvinotoolkit#599 from eizamaliev/demo_tests

Demo tests

ci/requirements-demos.txt

@@ -1,4 +1,7 @@
+joblib==0.14.0
 nibabel==2.5.1
 numpy==1.17.2 ; python_version >= "3.4"
+scikit-learn==0.21.3
 scipy==1.3.1
 six==1.12.0               # via nibabel
+tqdm==4.39.0

demos/mask_rcnn_demo/README.md

@@ -41,7 +41,7 @@ To run the demo, you can use public or pre-trained models. To download the pre-t
 
 You can use the following command to do inference on CPU on an image using a trained network:
 ```sh
-./mask_rcnn_demo -i <path_to_image>/inputImage.bmp -m <path_to_model>/faster_rcnn.xml
+./mask_rcnn_demo -i <path_to_image>/inputImage.bmp -m <path_to_model>/mask_rcnn_inception_resnet_v2_atrous_coco.xml
 ```
 
 ## Demo Output

demos/mask_rcnn_demo/models.lst

@@ -0,0 +1,5 @@
+# This file can be used with the --list option of the model downloader.
+mask_rcnn_inception_resnet_v2_atrous_coco
+mask_rcnn_inception_v2_coco
+mask_rcnn_resnet101_atrous_coco
+mask_rcnn_resnet50_atrous_coco

demos/python_demos/requirements.txt

@@ -1,3 +1,5 @@
-opencv-python
+nibabel
 numpy
+scikit-learn
 scipy
+tqdm

demos/tests/args.py

@@ -18,7 +18,8 @@
 from pathlib import Path
 
 ArgContext = collections.namedtuple('ArgContext',
-    ['source_dir', 'test_data_dir', 'dl_dir', 'model_info', 'image_sequences', 'image_sequence_dir'])
+    ['source_dir', 'test_data_dir', 'dl_dir', 'model_info', 'data_sequences', 'data_sequence_dir'])
+
 
 class TestDataArg:
     def __init__(self, rel_path):
@@ -27,9 +28,19 @@ def __init__(self, rel_path):
     def resolve(self, context):
         return str(context.test_data_dir / self.rel_path)
 
+
 def image_net_arg(id):
     return TestDataArg('ILSVRC2012_img_val/ILSVRC2012_val_{}.JPEG'.format(id))
 
+
+def brats_arg(id):
+    return TestDataArg('BraTS/{}'.format(id))
+
+
+def image_retrieval_arg(id):
+    return TestDataArg('Image_Retrieval/{}'.format(id))
+
+
 class ModelArg:
     def __init__(self, name, precision='FP32'):
         self.name = name
@@ -38,36 +49,39 @@ def __init__(self, name, precision='FP32'):
     def resolve(self, context):
         return str(context.dl_dir / context.model_info[self.name]["subdirectory"] / self.precision / (self.name + '.xml'))
 
-class ImagePatternArg:
+
+class DataPatternArg:
     def __init__(self, sequence_name):
         self.sequence_name = sequence_name
 
     def resolve(self, context):
-        seq_dir = context.image_sequence_dir / self.sequence_name
-        seq = [Path(image.resolve(context))
-            for image in context.image_sequences[self.sequence_name]]
+        seq_dir = context.data_sequence_dir / self.sequence_name
+        seq = [Path(data.resolve(context))
+            for data in context.data_sequences[self.sequence_name]]
 
-        assert len(set(image.suffix for image in seq)) == 1, "all images in the sequence must have the same extension"
+        assert len(set(data.suffix for data in seq)) == 1, "all images in the sequence must have the same extension"
         assert '%' not in seq[0].suffix
 
         name_format = 'input-%04d' + seq[0].suffix
 
         if not seq_dir.is_dir():
             seq_dir.mkdir(parents=True)
 
-            for index, image in enumerate(context.image_sequences[self.sequence_name]):
-                shutil.copyfile(image.resolve(context), str(seq_dir / (name_format % index)))
+            for index, data in enumerate(context.data_sequences[self.sequence_name]):
+                shutil.copyfile(data.resolve(context), str(seq_dir / (name_format % index)))
 
         return str(seq_dir / name_format)
 
-class ImageDirectoryArg:
+
+class DataDirectoryArg:
     def __init__(self, sequence_name):
-        self.backend = ImagePatternArg(sequence_name)
+        self.backend = DataPatternArg(sequence_name)
 
     def resolve(self, context):
         pattern = self.backend.resolve(context)
         return str(Path(pattern).parent)
 
+
 class DemoFileArg:
     def __init__(self, file_name):
         self.file_name = file_name

demos/tests/cases.py

@@ -16,7 +16,7 @@
 import sys
 
 from args import *
-from image_sequences import IMAGE_SEQUENCES
+from data_sequences import DATA_SEQUENCES
 
 MONITORS = {'-u': 'cdm'}
 TestCase = collections.namedtuple('TestCase', ['options'])
@@ -90,7 +90,7 @@ def single_option_cases(key, *args):
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImagePatternArg('person-vehicle-bike-detection-crossroad')}),
+            '-i': DataPatternArg('person-vehicle-bike-detection-crossroad')}),
         TestCase(options={'-m': ModelArg('person-vehicle-bike-detection-crossroad-0078')}),
         single_option_cases('-m_pa', None, ModelArg('person-attributes-recognition-crossroad-0230')),
         single_option_cases('-m_reid', None, ModelArg('person-reidentification-retail-0079')),
@@ -101,7 +101,7 @@ def single_option_cases(key, *args):
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImagePatternArg('gaze-estimation-adas')}),
+            '-i': DataPatternArg('gaze-estimation-adas')}),
         TestCase(options={
             '-m': ModelArg('gaze-estimation-adas-0002'),
             '-m_fd': ModelArg('face-detection-adas-0001'),
@@ -113,7 +113,7 @@ def single_option_cases(key, *args):
     NativeDemo(subdirectory='human_pose_estimation_demo', device_keys=['-d'], test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImagePatternArg('human-pose-estimation')}),
+            '-i': DataPatternArg('human-pose-estimation')}),
         TestCase(options={'-m': ModelArg('human-pose-estimation-0001')}),
     )),
 
@@ -122,7 +122,7 @@ def single_option_cases(key, *args):
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImagePatternArg('face-detection-adas')}),
+            '-i': DataPatternArg('face-detection-adas')}),
         TestCase(options={'-m': ModelArg('face-detection-adas-0001')}),
         [
             TestCase(options={}),
@@ -139,14 +139,21 @@ def single_option_cases(key, *args):
         ],
     )),
 
-    # TODO: mask_rcnn_demo: no models.lst
+    NativeDemo(subdirectory='mask_rcnn_demo', device_keys=['-d'], test_cases=combine_cases(
+        TestCase(options={'-i': DataDirectoryArg('semantic-segmentation-adas')}),
+        single_option_cases('-m',
+            ModelArg('mask_rcnn_inception_resnet_v2_atrous_coco'),
+            ModelArg('mask_rcnn_inception_v2_coco'),
+            ModelArg('mask_rcnn_resnet101_atrous_coco'),
+            ModelArg('mask_rcnn_resnet50_atrous_coco'))
+    )),
 
     NativeDemo(subdirectory='multi_channel/face_detection_demo',
             device_keys=['-d'],
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': IMAGE_SEQUENCES['face-detection-adas']}),
+            '-i': DATA_SEQUENCES['face-detection-adas']}),
         single_option_cases('-m',
             ModelArg('face-detection-adas-0001'),
             ModelArg('face-detection-adas-binary-0001', "INT1"),
@@ -159,7 +166,7 @@ def single_option_cases(key, *args):
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': IMAGE_SEQUENCES['human-pose-estimation'],
+            '-i': DATA_SEQUENCES['human-pose-estimation'],
             '-m': ModelArg('human-pose-estimation-0001')}),
     )),
 
@@ -168,15 +175,15 @@ def single_option_cases(key, *args):
         [
             TestCase(options={
                 '-m': ModelArg('face-detection-adas-0001'),
-                '-i': ImagePatternArg('face-detection-adas'),
+                '-i': DataPatternArg('face-detection-adas'),
             }),
             TestCase(options={
                 '-m': ModelArg('person-detection-retail-0002'),
-                '-i': ImagePatternArg('person-detection-retail'),
+                '-i': DataPatternArg('person-detection-retail'),
             }),
             TestCase(options={
                 '-m': ModelArg('person-detection-retail-0013'),
-                '-i': ImagePatternArg('person-detection-retail'),
+                '-i': DataPatternArg('person-detection-retail'),
             }),
         ],
     )),
@@ -186,7 +193,7 @@ def single_option_cases(key, *args):
     NativeDemo('pedestrian_tracker_demo', device_keys=['-d_det', '-d_reid'], test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImagePatternArg('person-detection-retail')}),
+            '-i': DataPatternArg('person-detection-retail')}),
         [
             TestCase(options={'-m_det': ModelArg('person-detection-retail-0002')}),
             TestCase(options={'-m_det': ModelArg('person-detection-retail-0013')}),
@@ -202,7 +209,7 @@ def single_option_cases(key, *args):
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImageDirectoryArg('vehicle-license-plate-detection-barrier')}),
+            '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}),
         TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0106')}),
         single_option_cases('-m_lpr',
             None,
@@ -215,11 +222,11 @@ def single_option_cases(key, *args):
         [
             TestCase(options={
                 '-m': ModelArg('road-segmentation-adas-0001'),
-                '-i': ImageDirectoryArg('road-segmentation-adas'),
+                '-i': DataDirectoryArg('road-segmentation-adas'),
             }),
             TestCase(options={
                 '-m': ModelArg('semantic-segmentation-adas-0001'),
-                '-i': ImageDirectoryArg('semantic-segmentation-adas'),
+                '-i': DataDirectoryArg('semantic-segmentation-adas'),
             }),
         ],
     )),
@@ -229,7 +236,7 @@ def single_option_cases(key, *args):
             test_cases=combine_cases(
         TestCase(options={'-no_show': None,
             **MONITORS,
-            '-i': ImagePatternArg('smart-classroom-demo'),
+            '-i': DataPatternArg('smart-classroom-demo'),
             '-m_fd': ModelArg('face-detection-adas-0001')}),
         [
             *combine_cases(
@@ -255,7 +262,7 @@ def single_option_cases(key, *args):
     )),
 
     NativeDemo(subdirectory='super_resolution_demo', device_keys=['-d'], test_cases=combine_cases(
-        TestCase(options={'-i': ImageDirectoryArg('single-image-super-resolution')}),
+        TestCase(options={'-i': DataDirectoryArg('single-image-super-resolution')}),
         TestCase(options={
             '-m': ModelArg('single-image-super-resolution-1033'),
         }),
@@ -264,17 +271,21 @@ def single_option_cases(key, *args):
     NativeDemo(subdirectory='text_detection_demo', device_keys=['-d_td', '-d_tr'], test_cases=combine_cases(
         TestCase(options={'-no_show': None, '-dt': 'video',
             **MONITORS,
-            '-i': ImagePatternArg('text-detection')}),
+            '-i': DataPatternArg('text-detection')}),
         single_option_cases('-m_td', ModelArg('text-detection-0003'), ModelArg('text-detection-0004')),
         single_option_cases('-m_tr', None, ModelArg('text-recognition-0012')),
     )),
 ]
 
 PYTHON_DEMOS = [
-    # TODO: 3d_segmentation_demo: no input data
+    PythonDemo(subdirectory='3d_segmentation_demo', device_keys=['-d'], test_cases=combine_cases(
+        TestCase(options={'-m': ModelArg('brain-tumor-segmentation-0001'),
+                          '-o': '.'}),
+        single_option_cases('-i', *DATA_SEQUENCES['brain-tumor-nifti']),
+    )),
 
     PythonDemo(subdirectory='action_recognition', device_keys=['-d'], test_cases=combine_cases(
-        TestCase(options={'--no_show': None, '-i': ImagePatternArg('action-recognition')}),
+        TestCase(options={'--no_show': None, '-i': DataPatternArg('action-recognition')}),
         [
             TestCase(options={
                 '-m_en': ModelArg('action-recognition-0001-encoder'),
@@ -287,12 +298,32 @@ def single_option_cases(key, *args):
         ],
     )),
 
-    # TODO: face_recognition_demo: requires face gallery
-    # TODO: image_retrieval_demo: current images does not suit the usecase, requires user defined gallery
+    PythonDemo(subdirectory='face_recognition_demo', device_keys=['-d_fd', '-d_lm', '-d_reid'],
+               test_cases=combine_cases(
+        TestCase(options={'--no_show': None,
+                          '-i': DataPatternArg('face-detection-adas'),
+                          '-fg': DataDirectoryArg('face-recognition-gallery')
+                          }),
+        single_option_cases('-m_fd',
+            ModelArg('face-detection-adas-0001'),
+            ModelArg('face-detection-adas-binary-0001', "INT1"),
+            ModelArg('face-detection-retail-0004'),
+            ModelArg('face-detection-retail-0005'),
+            ModelArg('face-detection-retail-0044')),
+        TestCase(options={'-m_lm': ModelArg('landmarks-regression-retail-0009')}),
+        TestCase(options={'-m_reid': ModelArg('face-reidentification-retail-0095')}),
+    )),
+
+    PythonDemo(subdirectory='image_retrieval_demo', device_keys=['-d'], test_cases=combine_cases(
+        TestCase(options={'--no_show':None,
+                          '-m': ModelArg('image-retrieval-0001')}),
+        single_option_cases('-i', *DATA_SEQUENCES['image-retrieval-video']),
+        single_option_cases('-g', image_retrieval_arg('gallery.txt')),
+    )),
 
     PythonDemo(subdirectory='instance_segmentation_demo', device_keys=[], test_cases=combine_cases(
         TestCase(options={'--no_show': None,
-            '-i': ImagePatternArg('instance-segmentation'),
+            '-i': DataPatternArg('instance-segmentation'),
             '--delay': '1',
             '-d': 'CPU',  # GPU is not supported
             '--labels': DemoFileArg('coco_labels.txt')}),
@@ -304,8 +335,8 @@ def single_option_cases(key, *args):
 
     PythonDemo(subdirectory='multi_camera_multi_person_tracking', device_keys=['-d'], test_cases=combine_cases(
         TestCase(options={'--no_show': None,
-            '-i': [ImagePatternArg('multi-camera-multi-person-tracking'),
-                ImagePatternArg('multi-camera-multi-person-tracking/repeated')],
+            '-i': [DataPatternArg('multi-camera-multi-person-tracking'),
+                DataPatternArg('multi-camera-multi-person-tracking/repeated')],
             '-m': ModelArg('person-detection-retail-0013')}),
         single_option_cases('--m_reid',
             ModelArg('person-reidentification-retail-0031'),
@@ -315,7 +346,7 @@ def single_option_cases(key, *args):
 
     PythonDemo(subdirectory='object_detection_demo_ssd_async', device_keys=['-d'], test_cases=combine_cases(
         TestCase(options={'--no_show': None,
-            '-i': ImagePatternArg('object-detection-demo-ssd-async')}),
+            '-i': DataPatternArg('object-detection-demo-ssd-async')}),
         single_option_cases('-m',
             ModelArg('face-detection-adas-0001'),
             ModelArg('face-detection-adas-binary-0001', "INT1"),
@@ -337,11 +368,11 @@ def single_option_cases(key, *args):
         [
             TestCase(options={
                 '-m': ModelArg('road-segmentation-adas-0001'),
-                '-i': IMAGE_SEQUENCES['road-segmentation-adas'],
+                '-i': DATA_SEQUENCES['road-segmentation-adas'],
             }),
             TestCase(options={
                 '-m': ModelArg('semantic-segmentation-adas-0001'),
-                '-i': IMAGE_SEQUENCES['semantic-segmentation-adas'],
+                '-i': DATA_SEQUENCES['semantic-segmentation-adas'],
             }),
         ],
     )),

demos/tests/image_sequences.py → demos/tests/data_sequences.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from args import image_net_arg
+from args import image_net_arg, brats_arg, image_retrieval_arg
 
-IMAGE_SEQUENCES = {
+DATA_SEQUENCES = {
     'action-recognition': [
         image_net_arg('00000001'),
         image_net_arg('00000002'),
@@ -38,6 +38,10 @@
         image_net_arg('00000020'),
     ],
 
+    'brain-tumor-nifti': [
+        brats_arg('BRATS_485.nii.gz'),
+    ],
+
     'face-detection-adas': [
         image_net_arg('00000002'),
         image_net_arg('00000032'),
@@ -51,6 +55,12 @@
         image_net_arg('00045630'),
     ],
 
+    'face-recognition-gallery': [
+        image_net_arg('00000184'),
+        image_net_arg('00008165'),
+        image_net_arg('00040548'),
+    ],
+
     'gaze-estimation-adas': [
         image_net_arg('00008165'),
         image_net_arg('00008170'),
@@ -77,6 +87,10 @@
         image_net_arg('00048311'),
     ],
 
+    'image-retrieval-video': [
+        image_retrieval_arg('4946fb41-9da0-4af7-a858-b443bee6d0f6.dav'),
+    ],
+
     'instance-segmentation': [
         image_net_arg('00000001'),
         image_net_arg('00000002'),