Skip to content

Commit

Permalink
Merge branch 'atlas'
Browse files Browse the repository at this point in the history
  • Loading branch information
@tampsa
tampsa committed Jan 23, 2024
2 parents 7ddbaf3 + e91c321 commit 9bbed39
Show file tree
Hide file tree
Showing 17 changed files with 2,124 additions and 114 deletions.
24 changes: 19 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,34 @@
.PHONY: all clean

CXX = g++
CXXFLAGS = -Wall -Wextra -O2 -std=c++11 -g
CXXFLAGS = -Wall -Wextra -O2 -std=c++17 -g

test_file_creation: util/test_file_creation.cc util/util.cc
$(CXX) $(CXXFLAGS) -o test_file_creation util/test_file_creation.cc util/util.cc -lkvazaar -lpthread

uvgrtp_sender: uvgrtp/sender.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/sender uvgrtp/sender.cc util/util.cc -luvgrtp -lpthread -lcryptopp
$(CXX) $(CXXFLAGS) -o uvgrtp/sender uvgrtp/sender.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_receiver: uvgrtp/receiver.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/receiver uvgrtp/receiver.cc util/util.cc -luvgrtp -lpthread -lcryptopp
$(CXX) $(CXXFLAGS) -o uvgrtp/receiver uvgrtp/receiver.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_latency_sender: uvgrtp/latency_sender.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/latency_sender uvgrtp/latency_sender.cc util/util.cc -luvgrtp -lpthread -lcryptopp
$(CXX) $(CXXFLAGS) -o uvgrtp/latency_sender uvgrtp/latency_sender.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_latency_receiver: uvgrtp/latency_receiver.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/latency_receiver uvgrtp/latency_receiver.cc util/util.cc -luvgrtp -lpthread -lcryptopp
$(CXX) $(CXXFLAGS) -o uvgrtp/latency_receiver uvgrtp/latency_receiver.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_vpcc_latency_sender: uvgrtp/vpcc_latency_sender.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/vpcc_latency_sender uvgrtp/vpcc_latency_sender.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_vpcc_latency_receiver: uvgrtp/vpcc_latency_receiver.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/vpcc_latency_receiver uvgrtp/vpcc_latency_receiver.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_vpcc_sender: uvgrtp/vpcc_sender.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/vpcc_sender uvgrtp/vpcc_sender.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

uvgrtp_vpcc_receiver: uvgrtp/vpcc_receiver.cc util/util.cc
$(CXX) $(CXXFLAGS) -o uvgrtp/vpcc_receiver uvgrtp/vpcc_receiver.cc util/util.cc uvgrtp/v3c_util.cc -luvgrtp -lpthread -lcryptopp

# ffmpeg_sender:
# $(CXX) $(CXXFLAGS) -Wno-unused -Wno-deprecated-declarations -Wno-unused-result -o ffmpeg/sender \
Expand Down Expand Up @@ -76,5 +88,7 @@ live555_latency_receiver: live555/latency_receiver.cc

clean:
rm -f uvgrtp/receiver uvgrtp/sender uvgrtp/latency_sender uvgrtp/latency_receiver \
uvgrtp/vpcc_latency_sender uvgrtp/vpcc_latency_receiver \
uvgrtp/vpcc_sender uvgrtp/vpcc_receiver \
ffmpeg/receiver ffmpeg/sender ffmpeg/latency_sender ffmpeg/latency_receiver \
live555/receiver live555/sender live555/latency test_file_creation
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ Latency receiver example:
--port 9999
```

The framework can also be used to benchmark transmission of Video-based Point Cloud Compression (V-PCC) files via uvgRTP. For this, specify the file format using `--format vpcc` for both sender and receiver and use a `.vpcc` file as the input. Both goodput and latency benchmarks support V-PCC files.

The latency results will only appear in the sending end. These too can be parsed into a summary with `parse.pl` script.

## Phase 4: Parsing the benchmark results
Expand Down
226 changes: 213 additions & 13 deletions benchmark.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,118 @@ sub recv_benchmark {
$socket->close();
}

sub vpcc_send_benchmark {
print "V-PCC benchmark sender\n";

my ($lib, $file, $saddr, $raddr, $port, $iter, $threads, $gen_recv, $e, $format, $srtp, @fps_vals) = @_;
my ($socket, $remote, $data);
my @execs = split ",", $e;

print "Waiting for receiver to connect to our TCP socket\n";

$socket = mk_ssock($saddr, $port);
$remote = $socket->accept();

my $result_directory = "./$lib/results";
print "Checking existance of results folder $result_directory\n";

unless(-e $result_directory or mkdir $result_directory) {
die "Unable to create $result_directory\n";
}

print "Starting send benchmark for $lib\n";

foreach (@execs) {
my $exec = $_;

unless(-e "./$lib/$exec") {
die "The executable ./$lib/$exec has not been created! \n";
}
foreach ((1 .. $threads)) {
my $thread = $_;
foreach (@fps_vals) {

my $fps = $_;
my $logname = "send_$format" . "_RTP" . "_$thread" . "threads_$fps". "fps_$iter" . "rounds";
if ($srtp)
{
$logname = "send_$format" . "_SRTP" . "_$thread" . "threads_$fps". "fps_$iter" . "rounds";
}

my $result_file = "$lib/results/$logname";

unlink $result_file if -e $result_file; # erase old results if they exist

for ((1 .. $iter)) {
print "Starting to benchmark sending at $fps fps, round $_\n";
$remote->recv($data, 16);
my $exit_code = system ("(time ./$lib/$exec $file $result_file $saddr $port $raddr $port $thread $fps $format $srtp) 2>> $result_file");
$remote->send("end") if $gen_recv;

die "Sender failed! \n" if ($exit_code ne 0);
}
}
}
}

print "V-PCC benchmark sender finished\n";
$socket->close();
}

sub vpcc_recv_benchmark {
print "V-PCC benchmark receiver\n";
my ($lib, $saddr, $raddr, $port, $iter, $threads, $e, $format, $srtp, @fps_vals) = @_;

print "Connecting to the TCP socket of the sender\n";
my $socket = mk_rsock($saddr, $port);
my @execs = split ",", $e;

my $result_directory = "./$lib/results";
print "Checking existance of results folder $result_directory\n";

unless(-e $result_directory or mkdir $result_directory) {
die "Unable to create $result_directory\n";
}

print "Starting receive benchmark for $lib\n";

foreach (@execs) {
my $exec = $_;

unless(-e "./$lib/$exec") {
die "The executable ./$lib/$exec has not been created! \n";
}

foreach ((1 .. $threads)) {
my $thread = $_;
foreach (@fps_vals) {
my $fps = $_;
my $logname = "recv_$format" . "_RTP" . "_$thread" . "threads_$fps". "fps_$iter" . "rounds";
if ($srtp)
{
$logname = "recv_$format" . "_SRTP" . "_$thread" . "threads_$fps". "fps_$iter" . "rounds";
}

my $result_file = "$lib/results/$logname";

unlink $result_file if -e $result_file; # erase old results if they exist

for ((1 .. $iter)) {
Time::HiRes::sleep(0.1); # sleep so packets from previous test don't interfere
print "Starting to benchmark receive at $fps fps, round $_\n";
$socket->send("start"); # I believe this is used to avoid firewall from blocking traffic
# please note that the local address for receiver is raddr
my $exit_code = system ("(time ./$lib/vpcc_receiver $result_file $raddr $port $saddr $port $thread $format $srtp) 2>> $result_file");
die "Receiver failed! \n" if ($exit_code ne 0);
}
}
}
}

print "V-PCC benchmark receiver finished\n";
$socket->close();
}

# use netcat to capture the stream
sub recv_generic {
print "Start netcat receiver\n";
Expand Down Expand Up @@ -256,6 +368,63 @@ sub recv_latency {
$socket->close();
}

sub vpcc_send_latency {

my ($lib, $file, $saddr, $raddr, $port, $fps, $iter, $format, $srtp) = @_;
my ($socket, $remote, $data);
print "VPCC latency send benchmark for $lib\n";

unless(-e "./$lib/vpcc_latency_sender") {
die "The executable ./$lib/latency_sender has not been created! \n";
}

$socket = mk_ssock($saddr, $port);
$remote = $socket->accept();

my $logname = "latencies_$format" . "_RTP_$fps". "fps_$iter" . "rounds";
if ($srtp)
{
$logname = "latencies_$format" . "_SRTP_$fps". "fps_$iter" . "rounds";
}



my $result_file = "$lib/results/$logname";
unlink $result_file if -e $result_file; # erase old results if they exist

for ((1 .. $iter)) {
print "Latency send benchmark round $_" . "/$iter\n";
$remote->recv($data, 16);

my $exit_code = system ("./$lib/vpcc_latency_sender $file $saddr $port $raddr $port $fps $format $srtp 2>> $result_file 2>&1");
die "Latency sender failed! \n" if ($exit_code ne 0);
}
print "VPCC latency send benchmark finished\n";
$socket->close();
}

sub vpcc_recv_latency {
my ($lib, $saddr, $raddr, $port, $iter, $format, $srtp) = @_;
print "VPCC latency receive benchmark for $lib\n";

unless(-e "./$lib/vpcc_latency_receiver") {
die "The executable ./$lib/latency_receiver has not been created! \n";
}

my $socket = mk_rsock($saddr, $port);

for ((1 .. $iter)) {
print "Latency receive benchmark round $_" . "/$iter\n";
sleep 1; # 1 s, make sure the sender has managed to catch up
$socket->send("start");

my $exit_code = system ("./$lib/vpcc_latency_receiver $raddr $port $saddr $port $format $srtp");
die "Latency receiver failed! \n" if ($exit_code ne 0);
}
print "VPCC latency receive benchmark finished\n";
$socket->close();
}

# TODO explain every parameter
sub print_help {
print "usage (benchmark):\n ./benchmark.pl \n"
Expand Down Expand Up @@ -317,7 +486,7 @@ sub print_help {


die "library not supported\n" if !grep (/$lib/, ("uvgrtp", "ffmpeg", "live555"));
die "format not supported\n" if !grep (/$format/, ("hevc", "vvc", "h265", "h266"));
die "format not supported\n" if !grep (/$format/, ("hevc", "vvc", "h265", "h266", "atlas", "vpcc"));

$fps = 30.0 if $lat and !$fps;

Expand Down Expand Up @@ -352,27 +521,58 @@ sub print_help {
}

if ($lat) {
system "make $lib" . "_latency_sender";
send_latency($lib, $file, $saddr, $raddr, $port, $fps, $iter, $format, $srtp);
if($format eq "vpcc") {
system "make $lib" . "_vpcc_latency_sender";
vpcc_send_latency($lib, $file, $saddr, $raddr, $port, $fps, $iter, $format, $srtp);
}
else {
system "make $lib" . "_latency_sender";
send_latency($lib, $file, $saddr, $raddr, $port, $fps, $iter, $format, $srtp);
}

} else {
if ($exec eq "default") {
system "make $lib" . "_sender";
$exec = "sender";
if($format eq "vpcc") {
if ($exec eq "default") {
system "make $lib" . "_vpcc_sender";
$exec = "vpcc_sender";
}
vpcc_send_benchmark($lib, $file, $saddr, $raddr, $port, $iter, $threads, $nc, $exec, $format, $srtp, @fps_vals);
}
else {
if ($exec eq "default") {
system "make $lib" . "_sender";
$exec = "sender";
}
send_benchmark($lib, $file, $saddr, $raddr, $port, $iter, $threads, $nc, $exec, $format, $srtp, @fps_vals);
}
send_benchmark($lib, $file, $saddr, $raddr, $port, $iter, $threads, $nc, $exec, $format, $srtp, @fps_vals);
}
} elsif ($role eq "recv" or $role eq "receive" or $role eq "receiver") {
die "Please specify test format with --format for receiver" if !$format;

if ($lat) {
system "make $lib" . "_latency_receiver";
recv_latency($lib, $saddr, $raddr, $port, $iter, $format, $srtp);
if($format eq "vpcc") {
system "make $lib" . "_vpcc_latency_receiver";
vpcc_recv_latency($lib, $saddr, $raddr, $port, $iter, $format, $srtp);
}
else {
system "make $lib" . "_latency_receiver";
recv_latency($lib, $saddr, $raddr, $port, $iter, $format, $srtp);
}
} elsif (!$nc) {
if ($exec eq "default") {
system "make $lib" . "_receiver";
$exec = "receiver";
if($format eq "vpcc") {
if ($exec eq "default") {
system "make $lib" . "_vpcc_receiver";
$exec = "vpcc_receiver";
}
vpcc_recv_benchmark($lib, $saddr, $raddr, $port, $iter, $threads, $exec, $format, $srtp, @fps_vals);
}
else {
if ($exec eq "default") {
system "make $lib" . "_receiver";
$exec = "receiver";
}
recv_benchmark($lib, $saddr, $raddr, $port, $iter, $threads, $exec, $format, $srtp, @fps_vals);
}
recv_benchmark($lib, $saddr, $raddr, $port, $iter, $threads, $exec, $format, $srtp, @fps_vals);
} else {
recv_generic($lib, $saddr, $port, $iter, $threads, @fps_vals);
}
Expand Down
9 changes: 5 additions & 4 deletions parse.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ sub parse {
}

sub parse_latency {
my ($lib, $path, $unit) = @_;
my ($lib, $path, $nframes, $unit) = @_;
my ($frames, $avg, $intra, $inter, $cnt) = (0) x 5;
my $frame_avg = 0;

Expand All @@ -391,7 +391,7 @@ sub parse_latency {
while (my $line = <$fh>) {
my @nums = ($line =~ m/(\d+).*intra\s(\d+\.\d+).*inter\s(\d+\.\d+).*avg\s(\d+\.\d+)/);

if ($nums[0] == 598) {
if ($nums[0] == $nframes) {
$frame_avg++;
}

Expand All @@ -406,7 +406,7 @@ sub parse_latency {
$intra /= $cnt;
$inter /= $cnt;
$avg /= $frame_avg;
$frames = 100*$frames/(598*$rounds);
$frames = 100*$frames/($nframes*$rounds);

print "Completed: $frames%, intra $intra ms, inter $inter ms, avg $avg ms\n";
}
Expand Down Expand Up @@ -440,6 +440,7 @@ sub print_help {
GetOptions(
"library|lib|l=s" => \(my $lib = ""),
"role|r=s" => \(my $role = ""),
"nframes" => \(my $nframes = 0),
"path|dir|directory|p=s" => \(my $path = ""),
"threadst|threads|=i" => \(my $threads = 0),
"iter|iterations|rounds|i=i" => \(my $iter = 0),
Expand Down Expand Up @@ -470,7 +471,7 @@ sub print_help {
} elsif ($parse eq "csv") {
parse_csv($lib, $iter, $path, $unit, $filesize);
} elsif ($parse eq "latency") {
parse_latency($lib, $path, $unit);
parse_latency($lib, $path, $nframes, $unit);
} elsif ($role eq "send") {
print_send($lib, $iter, $threads, $path, $unit, $filesize);
} elsif ($role eq "recv") {
Expand Down
Loading

0 comments on commit 9bbed39

Please sign in to comment.