starting

.gitignore

@@ -0,0 +1 @@
+data/taichi1

README.md

@@ -1 +1,8 @@
-# DeepDance
+
+# Objectif de ce code
+(Ce code est indépendant de tout le reste du dépôt)
+
+À partir d'une vidéo d'une personne source et d'une autre d'une personne, notre objectif est de générer une nouvelle vidéo de la cible effectuant les mêmes mouvements que la source. 
+
+[Allez voir le sujet du TP ici](http://alexandre.meyer.pages.univ-lyon1.fr/m2-apprentissage-profond-image/am/tp_dance/)
+

data/taichi_crawle_todo.py

@@ -0,0 +1,115 @@
+#check https://github.com/pytube/pytube
+
+import numpy as np
+import pandas as pd
+import imageio
+import os
+import subprocess
+from multiprocessing import Pool
+from itertools import cycle
+import warnings
+import glob
+import time
+from tqdm import tqdm
+from argparse import ArgumentParser
+from skimage import img_as_ubyte
+from skimage.transform import resize
+warnings.filterwarnings("ignore")
+
+DEVNULL = open(os.devnull, 'wb')
+
+
+def save(path, frames, format):
+    if format == '.mp4':
+        imageio.mimsave(path, frames)
+    elif format == '.png':
+        if os.path.exists(path):
+            print ("Warning: skiping video %s" % os.path.basename(path))
+            return
+        else:
+            os.makedirs(path)
+        for j, frame in enumerate(frames):
+            imageio.imsave(os.path.join(path, str(j).zfill(7) + '.png'), frames[j]) 
+    else:
+        print ("Unknown format %s" % format)
+        exit()
+
+
+def download(video_id, args):
+    video_path = os.path.join(args.video_folder, video_id + ".mp4")
+    subprocess.call([args.youtube, '-f', "''best/mp4''", '--write-auto-sub', '--write-sub',
+                     '--sub-lang', 'en', '--skip-unavailable-fragments',
+                     "https://www.youtube.com/watch?v=" + video_id, "--output",
+                     video_path], stdout=DEVNULL, stderr=DEVNULL)
+    return video_path
+
+
+def run(data):
+    video_id, args = data
+    if not os.path.exists(os.path.join(args.video_folder, video_id.split('#')[0] + '.mp4')):
+       download(video_id.split('#')[0], args)
+
+    if not os.path.exists(os.path.join(args.video_folder, video_id.split('#')[0] + '.mp4')):
+       print ('Can not load video %s, broken link' % video_id.split('#')[0])
+       return 
+    reader = imageio.get_reader(os.path.join(args.video_folder, video_id.split('#')[0] + '.mp4'))
+    fps = reader.get_meta_data()['fps']
+
+    df = pd.read_csv(args.metadata)
+    df = df[df['video_id'] == video_id]
+
+    all_chunks_dict = [{'start': df['start'].iloc[j], 'end': df['end'].iloc[j],
+                        'bbox': list(map(int, df['bbox'].iloc[j].split('-'))), 'frames':[]} for j in range(df.shape[0])]
+    ref_fps = df['fps'].iloc[0]
+    ref_height = df['height'].iloc[0]
+    ref_width = df['width'].iloc[0]
+    partition = df['partition'].iloc[0]
+    try:
+        for i, frame in enumerate(reader):
+            for entry in all_chunks_dict:
+                if (i * ref_fps >= entry['start'] * fps) and (i * ref_fps < entry['end'] * fps):
+                    left, top, right, bot = entry['bbox']
+                    left = int(left / (ref_width / frame.shape[1]))
+                    top = int(top / (ref_height / frame.shape[0]))
+                    right = int(right / (ref_width / frame.shape[1]))
+                    bot = int(bot / (ref_height / frame.shape[0]))
+                    crop = frame[top:bot, left:right]
+                    if args.image_shape is not None:
+                       crop = img_as_ubyte(resize(crop, args.image_shape, anti_aliasing=True))
+                    entry['frames'].append(crop)
+    except imageio.core.format.CannotReadFrameError:
+        None
+
+    for entry in all_chunks_dict:
+        first_part = '#'.join(video_id.split('#')[::-1])
+        path = first_part + '#' + str(entry['start']).zfill(6) + '#' + str(entry['end']).zfill(6) + '.mp4'
+        save(os.path.join(args.out_folder, partition, path), entry['frames'], args.format)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--video_folder", default='youtube-taichi', help='Path to youtube videos')
+    parser.add_argument("--metadata", default='taichi-metadata-new.csv', help='Path to metadata')
+    parser.add_argument("--out_folder", default='taichi-png', help='Path to output')
+    parser.add_argument("--format", default='.png', help='Storing format')
+    parser.add_argument("--workers", default=1, type=int, help='Number of workers')
+    parser.add_argument("--youtube", default='./youtube-dl', help='Path to youtube-dl')
+
+    parser.add_argument("--image_shape", default=(256, 256), type=lambda x: tuple(map(int, x.split(','))),
+                        help="Image shape, None for no resize")
+
+    args = parser.parse_args()
+    if not os.path.exists(args.video_folder):
+        os.makedirs(args.video_folder)
+    if not os.path.exists(args.out_folder):
+        os.makedirs(args.out_folder)
+    for partition in ['test', 'train']:
+        if not os.path.exists(os.path.join(args.out_folder, partition)):
+            os.makedirs(os.path.join(args.out_folder, partition))
+
+    df = pd.read_csv(args.metadata)
+    video_ids = set(df['video_id'])
+    pool = Pool(processes=args.workers)
+    args_list = cycle([args])
+    for chunks_data in tqdm(pool.imap_unordered(run, zip(video_ids, args_list))):
+        None  

src/DanceDemo.py

@@ -0,0 +1,76 @@
+import numpy as np
+import cv2
+import os
+import pickle
+import sys
+
+import tp
+from VideoSkeleton import VideoSkeleton
+from VideoSkeleton import combineTwoImages
+from VideoReader import VideoReader
+from Skeleton import Skeleton
+from GenNearest import GenNeirest
+from GenVanillaNN import *
+from GenGAN import *
+
+
+class DanceDemo:
+    """ class that run a demo of the dance.
+        The animation/posture from self.source is applied to character define self.target using self.gen
+    """
+    def __init__(self, filename_src, typeOfGen=2):
+        self.target = VideoSkeleton( "tp/dance/data/taichi1.mp4" )
+        self.source = VideoReader(filename_src)
+        if typeOfGen==1:           # Nearest
+            print("Generator: GenNeirest")
+            self.generator = GenNeirest(self.target)
+        # elif typeOfGen==2:         # VanillaNN
+        #     print("Generator: GenSimpleNN")
+        #     self.generator = GenVanillaNN( self.target, loadFromFile=True, optSkeOrImage=1)
+        # elif typeOfGen==3:         # VanillaNN
+        #     print("Generator: GenSimpleNN")
+        #     self.generator = GenVanillaNN( self.target, loadFromFile=True, optSkeOrImage=2)
+        # elif typeOfGen==4:         # GAN
+        #     print("Generator: GenSimpleNN")
+        #     self.generator = GenGAN( self.target, loadFromFile=True)
+        else:
+            print("DanceDemo: typeOfGen error!!!")
+
+
+    def draw(self):
+        ske = Skeleton()
+        image_err = np.zeros((256, 256, 3), dtype=np.uint8)
+        image_err[:, :] = (0, 0, 255)  # (B, G, R)
+        for i in range(self.source.getTotalFrames()):
+            image_src = self.source.readFrame()
+            if i%5 == 0:
+                isSke, image_src, ske = self.target.cropAndSke(image_src, ske)
+                if isSke:
+                    ske.draw(image_src)
+                    image_tgt = self.generator.generate(ske)            # GENERATOR !!!
+                    image_tgt = image_tgt*255
+                    image_tgt = cv2.resize(image_tgt, (128, 128))
+                else:
+                    image_tgt = image_err
+                image_combined = combineTwoImages(image_src, image_tgt)
+                image_combined = cv2.resize(image_combined, (512, 256))
+                cv2.imshow('Image', image_combined)
+                key = cv2.waitKey(1)
+                if key & 0xFF == ord('q'):
+                    break
+                if key & 0xFF == ord('n'):
+                    self.source.readNFrames( 100 )
+        cv2.destroyAllWindows()
+
+
+
+if __name__ == '__main__':
+    # NEAREST = 1
+    # VANILLA_NN_SKE = 2
+    # VANILLA_NN_Image = 3
+    # GAN = 4
+    GEN_TYPE = 1
+    ddemo = DanceDemo("data/taichi2_full.mp4", GEN_TYPE)
+    #ddemo = DanceDemo("data/taichi1.mp4")
+    #ddemo = DanceDemo("data/karate1.mp4")
+    ddemo.draw()

src/GenGAN.py

@@ -0,0 +1,111 @@
+
+import numpy as np
+import cv2
+import os
+import pickle
+import sys
+import math
+
+import matplotlib.pyplot as plt
+
+from torchvision.io import read_image
+import torch.nn as nn
+import torch.nn.functional as F
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms
+from torch.utils.tensorboard import SummaryWriter
+
+from VideoSkeleton import VideoSkeleton
+from VideoReader import VideoReader
+from Skeleton import Skeleton
+from GenVanillaNN import * 
+
+
+
+class Discriminator(nn.Module):
+    def __init__(self, ngpu=0):
+        super(Discriminator, self).__init__()
+        self.ngpu = ngpu
+
+
+    def forward(self, input):
+        pass
+        #return self.model(input)
+
+
+
+
+class GenGAN():
+    """ class that Generate a new image from videoSke from a new skeleton posture
+       Fonc generator(Skeleton)->Image
+    """
+    def __init__(self, videoSke, loadFromFile=False):
+        self.netG = GenNNSkeToImage()
+        self.netD = Discriminator()
+        self.real_label = 1.
+        self.fake_label = 0.
+        self.filename = 'data/Dance/DanceGenGAN.pth'
+        tgt_transform = transforms.Compose(
+                            [transforms.Resize((64, 64)),
+                            #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+                            transforms.CenterCrop(64),
+                            transforms.ToTensor(),
+                            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+                            ])
+        self.dataset = VideoSkeletonDataset(videoSke, ske_reduced=True, target_transform=tgt_transform)
+        self.dataloader = torch.utils.data.DataLoader(dataset=self.dataset, batch_size=32, shuffle=True)
+        if loadFromFile and os.path.isfile(self.filename):
+            print("GenGAN: Load=", self.filename, "   Current Working Directory=", os.getcwd())
+            self.netG = torch.load(self.filename)
+
+
+    def train(self, n_epochs=20):
+        pass
+
+
+
+
+    def generate(self, ske):           # TP-TODO
+        """ generator of image from skeleton """
+        pass
+        # ske_t = torch.from_numpy( ske.__array__(reduced=True).flatten() )
+        # ske_t = ske_t.to(torch.float32)
+        # ske_t = ske_t.reshape(1,Skeleton.reduced_dim,1,1) # ske.reshape(1,Skeleton.full_dim,1,1)
+        # normalized_output = self.netG(ske_t)
+        # res = self.dataset.tensor2image(normalized_output[0])
+        # return res
+
+
+
+
+if __name__ == '__main__':
+    force = False
+    if len(sys.argv) > 1:
+        filename = sys.argv[1]
+        if len(sys.argv) > 2:
+            force = sys.argv[2].lower() == "true"
+    else:
+        filename = "data/taichi1.mp4"
+    print("GenGAN: Current Working Directory=", os.getcwd())
+    print("GenGAN: Filename=", filename)
+
+    targetVideoSke = VideoSkeleton(filename)
+
+    #if False:
+    if True:    # train or load
+        # Train
+        gen = GenGAN(targetVideoSke, False)
+        gen.train(4) #5) #200)
+    else:
+        gen = GenGAN(targetVideoSke, loadFromFile=True)    # load from file        
+
+
+    for i in range(targetVideoSke.skeCount()):
+        image = gen.generate(targetVideoSke.ske[i])
+        #image = image*255
+        nouvelle_taille = (256, 256) 
+        image = cv2.resize(image, nouvelle_taille)
+        cv2.imshow('Image', image)
+        key = cv2.waitKey(-1)
+

src/GenNearest.py

@@ -0,0 +1,31 @@
+
+import numpy as np
+import cv2
+import os
+import pickle
+import sys
+import math
+
+from VideoSkeleton import VideoSkeleton
+from VideoReader import VideoReader
+from Skeleton import Skeleton
+
+
+
+class GenNeirest:
+    """ class that Generate a new image from videoSke from a new skeleton posture
+       Fonc generator(Skeleton)->Image
+       Neirest neighbor method: it select the image in videoSke that has the skeleton closest to the skeleton
+    """
+    def __init__(self, videoSkeTgt):
+        self.videoSkeletonTarget = videoSkeTgt
+
+    def generate(self, ske):           
+        """ generator of image from skeleton """
+        # TP-TODO
+        empty = np.ones((64,64, 3), dtype=np.uint8)
+        return empty
+
+
+
+