adding the metric section

scripts/metrics/.gitignore

@@ -0,0 +1 @@
+results/

scripts/metrics/add_compute.py

@@ -27,26 +27,26 @@
 import copy 
 from pyquaternion import Quaternion
 import pickle 
-import visii 
+import nvisii as visii 
 import subprocess 
 
 
 
 parser = argparse.ArgumentParser()
 
 parser.add_argument('--data_prediction', 
-    default = "dope_annotation/grocery_dope_dr/", 
+    default = "data/table_dope_results/", 
     help='path to prediction data')
 parser.add_argument('--data', 
-    default="/home/jtremblay/code/hope-dataset/images/", 
+    default="data/table_ground_truth/", 
     help='path to data ground truth')
 parser.add_argument('--models', 
     # /home/jtremblay/code/nvdu/nvdu/data/ycb/aligned_cm/AlphabetSoup/google_16k
     # default="/home/jtremblay/code/nvdu/nvdu/data/ycb/original/", 
-    default="/home/jtremblay/code/visii_dr/content/models/grocery/", 
+    default="content/", 
     help='path to the 3D grocery models')
 parser.add_argument("--outf",
-    default=None,
+    default="results/",
     help="where to put the data"
     )
 parser.add_argument('--adds',
@@ -69,6 +69,10 @@
 if opt.outf is None:
     opt.outf = opt.data_prediction
 
+if not os.path.isdir(opt.outf):
+    print(f'creating the folder: {opt.outf}')
+    os.mkdir(opt.outf)
+
 if os.path.isdir(opt.outf + "/tmp"):
     print(f'folder {opt.outf + "/tmp"}/ exists')
 else:
@@ -117,7 +121,7 @@ def create_obj(
     path_tex = None,
     scale = 1, 
     rot_base = None, #visii quat
-    pos_base = None, # visii vec3
+    pos_base = (-10,-10,-10), # visii vec3
     ):
 
 
@@ -159,10 +163,9 @@ def create_obj(
         obj_entity.get_transform().set_rotation(rot_base)
     if not pos_base is None:
         obj_entity.get_transform().set_position(pos_base)
-
-    if 'cherries' in name.lower():
-        print(obj_entity.get_mesh().get_min_aabb_corner()*-1 + obj_entity.get_mesh().get_max_aabb_corner())
+    print(f' created: {obj_entity.get_name()}')
     return obj_entity
+
 create_obj.meshes = {}
 create_obj.textures = {}
 
@@ -197,7 +200,7 @@ def add_cuboid(name, debug=False):
     for i_p, p in enumerate(cuboid):
         child_transform = visii.transform.create(f"{name}_cuboid_{i_p}")
         child_transform.set_position(p)
-        child_transform.set_scale(visii.vec3(0.3))
+        child_transform.set_scale(visii.vec3(0.1))
         child_transform.set_parent(obj.get_transform())
         if debug: 
             visii.entity.create(
@@ -218,13 +221,12 @@ def add_cuboid(name, debug=False):
 def get_models(path,suffix=""):
     models = {}
     for folder in glob.glob(path+"/*/"):
-        # if 'Ketchup' not in folder:
-        #     continue
+
         model_name = folder.replace(path,"").replace('/',"")
         print('loading',model_name + suffix)
         models[model_name] = create_obj(
             name = model_name + suffix,
-            path_obj = folder + "/google_16k/textured_simple.obj",
+            path_obj = folder + "/google_16k/textured.obj",
             path_tex = folder + "/google_16k/texture_map_flat.png",
             scale = 0.01
         )
@@ -245,9 +247,9 @@ def get_models(path,suffix=""):
 # data_thruth = get_all_entries(opt.data,'scene_all_realsense.json')
 # if len(data_thruth) == 0: 
 #     data_thruth = get_all_entries(opt.data,'scene_realsense.json')
-data_thruth = get_all_entries(opt.data,"*realsense_rgb.json")
-data_prediction = get_all_entries(opt.data_prediction,"*realsense_rgb.json")
-data_prediction += get_all_entries(opt.data_prediction,"*realsense_rgb..json")
+data_thruth = get_all_entries(opt.data,"*.json")
+data_prediction = get_all_entries(opt.data_prediction,"*.json")
+
 
 print('number of ground thruths found',len(data_thruth))
 print("number of predictions found",len(data_prediction))
@@ -282,7 +284,7 @@ def get_models(path,suffix=""):
 
     if pred_scene is None:
         continue
-    print(gt_file)
+    # print(gt_file)
     gt_json = None
     with open(gt_file) as json_file:
         gt_json = json.load(json_file)
@@ -297,6 +299,10 @@ def get_models(path,suffix=""):
     for obj in gt_json['objects']:
 
         name_gt = obj['class']
+
+        # little hack from bug in the data
+        if name_gt == '003':
+            name_gt = "003_cracker_box_16k"
         objects_gt.append(
             [
                 name_gt,
@@ -308,9 +314,9 @@ def get_models(path,suffix=""):
                         obj['quaternion_xyzw'][2],
                     ),
                     "position":visii.vec3(
-                        obj['location'][0]/100,
-                        obj['location'][1]/100,
-                        obj['location'][2]/100,
+                        obj['location'][0],
+                        obj['location'][1],
+                        obj['location'][2],
                     )
                 }
             ]
@@ -326,7 +332,8 @@ def get_models(path,suffix=""):
     for obj_guess in gu_json['objects']:
 
         name_guess = obj_guess['class']
-        name_look_up = obj_guess['class'].split('_')[0]
+        # name_look_up = obj_guess['class'].split("_")[0]
+        name_look_up = obj_guess['class']
 
         # need to add rotation for DOPE prediction, if your frames are aligned 
         try:
@@ -336,7 +343,10 @@ def get_models(path,suffix=""):
                     float(obj_guess['quaternion_xyzw'][0]),
                     float(obj_guess['quaternion_xyzw'][1]),
                     float(obj_guess['quaternion_xyzw'][2]),
-                ) * visii.angleAxis(1.57, visii.vec3(1,0,0)) * visii.angleAxis(1.57, visii.vec3(0,0,1))
+                ) 
+                # * visii.angleAxis(1.57, visii.vec3(1,0,0)) * visii.angleAxis(1.57, visii.vec3(0,0,1))
+                # * visii.angleAxis(1.57*2, visii.vec3(0,0,1)) 
+                # * visii.angleAxis(1.57, visii.vec3(0,1,0))
                 ,
                 "position":visii.vec3(
                     float(str(obj_guess['location'][0]))/100.0,
@@ -373,6 +383,8 @@ def get_models(path,suffix=""):
         for i_obj_gt, obj_gt in enumerate(objects_gt):
             name_gt, pose_mesh_gt = obj_gt
 
+            # print(name_look_up,name_gt)
+
             if name_look_up == name_gt:
                 candidates.append([i_obj_gt, pose_mesh_gt, name_gt])
 
@@ -384,17 +396,25 @@ def get_models(path,suffix=""):
             i_gt, pose_gt, name_gt = candi_gt
             # if i_gt in used_index:
             #     continue
+            # print(meshes_gt.keys())
             visii_gt = meshes_gt[name_gt]
 
             visii_gt.get_transform().set_position(pose_gt['position'])
             visii_gt.get_transform().set_rotation(pose_gt['rotation'])
 
+            # visii_gt.get_transform().set_position(visii.vec3(-10,-10,-10))
+            # visii_gt.get_transform().set_rotation(pose_gt['rotation'])
+
+
             visii_gu = meshes_gu[name_look_up]
 
             visii_gu.get_transform().set_position(pose_mesh['position'])
             visii_gu.get_transform().set_rotation(pose_mesh['rotation'])
 
+            # dope is in the opencv frame, need to be put in the opengl frame
+            visii_gu.get_transform().rotate_around(visii.vec3(0,0,0),visii.angleAxis(visii.pi(), visii.vec3(1,0,0)))
 
+
             if opt.adds:
                 if opt.cuboid:                                            
                     dist = 0
@@ -455,7 +475,6 @@ def get_models(path,suffix=""):
                             )
 
                     dist /= 9
-
                 else:
                     dist = []
                     vertices = visii_gt.get_mesh().get_vertices()
@@ -507,9 +526,9 @@ def get_models(path,suffix=""):
 labels = []
 data = []
 for key in adds_objects.keys():
-    pickle.dump(adds_objects[key],open(f'{opt.outf}/adds_{key.split("_")[0]}.p','wb'))
+    pickle.dump(adds_objects[key],open(f'{opt.outf}/adds_{key}.p','wb'))
     labels.append(key)
-    data.append(f'{opt.outf}/adds_{key.split("_")[0]}.p')
+    data.append(f'{opt.outf}/adds_{key}.p')
 
 
 array_to_call = ["python", "make_graphs.py","--outf", opt.outf,'--labels']

scripts/metrics/download_content.sh

@@ -0,0 +1,28 @@
+mkdir data 
+
+cd data 
+
+wget https://www.dropbox.com/s/qeljw3vjnc416bs/table_003_cracker_box_dope_results.zip
+wget https://www.dropbox.com/s/mn2yqflc6fcqaic/table_003_cracker_box.zip
+
+unzip table_003_cracker_box_dope_results.zip 
+rm table_003_cracker_box_dope_results.zip
+mkdir table_dope_results/
+mv table_003_cracker_box table_dope_results/scene1/
+
+unzip table_003_cracker_box.zip
+rm table_003_cracker_box.zip
+mkdir table_ground_truth/
+mv table_003_cracker_box table_ground_truth/scene1/
+
+cd ../
+
+mkdir content
+cd content
+
+wget https://www.dropbox.com/s/b61es9q5nhwtooi/003_cracker_box.zip
+unzip 003_cracker_box.zip
+rm 003_cracker_box.zip
+mv 003_cracker_box 003_cracker_box_16k
+
+cd ../

scripts/metrics/make_graphs.py

@@ -85,7 +85,7 @@
     print(file)
     label = file.split("/")[-1]
     label = label.replace('adds_','').replace(".p",'')
-    filename = label.split("_")[0] 
+    filename = label
 
     if not counts_dict is None: 
         fig = plt.figure()

scripts/metrics/readme.md

@@ -0,0 +1,38 @@
+# ADD metrics computation and figure
+
+## Requirements
+
+Run the download content file: `./download_content.sh`, this downloads a simple scene with annotation rendered by NViSII and with DOPE predictions.
+
+## How to run
+
+If you downloaded the previous content you can execute the following: 
+
+```
+python add_compute.py
+```
+and you should get the following displayed:
+```
+mean 0.0208515107260977 std 0.016006083915162977 ratio 17/22
+auc at  0.02 : 0.5
+auc at  0.04 : 0.6818181818181818
+auc at  0.06 : 0.7272727272727273
+auc 0.6115249999999999
+```
+This means the area under the curve, *auc* from 0 cm to 10 cm is 0.61. This script also produces graphs such as: 
+
+![example of graph](results/output.png)
+
+These are the metrics we reported in the original DOPE paper. I will refer to the paper for explaining the graph. 
+
+## Assumptions
+We make a few assumptions in this script. 
+1. We assume the folders structures are the same and there are only scenes in the folder. See `data/` folder example from downloading the content. 
+2. We assume the notation folder is in the opengl format and that it is using the nvisii outputs from the data generation pipeline. If you use a diffirent file format please update the script or your data. 
+3. We assume the inferences are from DOPE inference, _e.g._, the poses are in the opengl format. These conventions are easy to change, _e.g._, look for the line `visii_gu.get_transform().rotate_around` in `add_compute.py` to change the pose convention.
+
+If the script takes to long to run, please run with `--cuboid`, instead of using the 3d models vertices to compare the metric, it uses the 3d cuboid of the 3d model to compute the metric.
+
+# TODO 
+- Make a `requirement.txt` file. 
+- Possibly subsamble vertices so computation is faster

scripts/nvisii_data_gen/.gitignore

@@ -0,0 +1 @@
+output/

scripts/nvisii_data_gen/generate_dataset.py

@@ -5,13 +5,19 @@
 
 # 20 000 images
 
-for i in range(0, 100):
+for i in range(97, 100):
 	to_call = [
 		"python",'single_video_pybullet.py',
 		'--spp','10',
 		'--nb_frames', '200',
-		'--nb_objects',str(int(random.uniform(50,75))),
-		'--scale', '0.01',
-		'--outf',f"dataset/{str(i).zfill(3)}",
+		'--nb_distractors','10',
+		'--nb_objects',str(int(random.uniform(2,5))),
+		'--scale', '0.015',
+		'--skyboxes_folder','/home/jtremblay/code/visii_dr/dome_hdri_haven/',
+		"--skip_frame",'100',
+		'--objs_folder_distrators','/media/jtremblay/data_large/google_scanned/google_scanned_models/',
+		'--path_single_obj','/home/jtremblay/code/visii_dr/content/models/grocery_ycb/003_cracker_box/google_16k/textured.obj',
+		'--outf',f"andrew_ycb_dope/{str(i).zfill(3)}",
 	]
 	subprocess.call(to_call)
+	# raise()

scripts/nvisii_data_gen/single_video_pybullet.py

@@ -270,8 +270,6 @@ def adding_mesh_object(name, obj_to_load, texture_to_load, model_info_path=None,
     # obj_to_load = toy_to_load + "/meshes/model.obj"
     # texture_to_load = toy_to_load + "/materials/textures/texture.png"
 
-    print("loading:", obj_to_load)
-
     if texture_to_load is None:
         toys = load_obj_scene(obj_to_load)
 
@@ -290,6 +288,7 @@ def adding_mesh_object(name, obj_to_load, texture_to_load, model_info_path=None,
                 material = toy_material,
                 mesh = visii.mesh.create_from_file(name,obj_to_load),
             )
+
         toy_transform = obj_export.get_transform()
         obj_export.get_material().set_roughness(random.uniform(0.1, 0.5))
 
@@ -404,7 +403,7 @@ def adding_mesh_object(name, obj_to_load, texture_to_load, model_info_path=None,
         model_info_path = os.path.dirname(opt.path_single_obj) + '/model_info.json'
         adding_mesh_object(f"single_obj_{i_object}",
                            opt.path_single_obj,
-                           None,
+                           "/home/jtremblay/code/visii_dr/content/models/grocery_ycb/003_cracker_box/google_16k/texture_map_flat.png",
                            model_info_path,
                            scale=opt.scale,
                            debug=opt.debug)