Skip to content

Commit 912c982

Browse files
shixingshixing
committed
add translate sampled data and translate scripts
1 parent 0bda57c commit 912c982

10 files changed

Lines changed: 38277 additions & 57 deletions

‎sample_data/dev_english.txt.tok.lc‎

Lines changed: 6003 additions & 0 deletions
Large diffs are not rendered by default.

‎sample_data/dev_french.txt.tok.lc‎

Lines changed: 6003 additions & 0 deletions
Large diffs are not rendered by default.

‎sample_data/test_english.tok.lc‎

Lines changed: 3003 additions & 0 deletions
Large diffs are not rendered by default.

‎sample_data/test_french.tok.lc‎

Lines changed: 3003 additions & 0 deletions
Large diffs are not rendered by default.

‎sample_data/train_english.txt.tok.lc.10k‎

Lines changed: 10000 additions & 0 deletions
Large diffs are not rendered by default.

‎sample_data/train_french.txt.tok.lc.10k‎

Lines changed: 10000 additions & 0 deletions
Large diffs are not rendered by default.

‎scripts/compile.sh.bk‎

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎scripts/multi-bleu.perl‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
#!/usr/bin/env perl
2+
#
3+
# This file is part of moses. Its use is licensed under the GNU Lesser General
4+
# Public License version 2.1 or, at your option, any later version.
5+
6+
# $Id$
7+
use warnings;
8+
use strict;
9+
10+
my $lowercase = 0;
11+
if ($ARGV[0] eq "-lc") {
12+
$lowercase = 1;
13+
shift;
14+
}
15+
16+
my $stem = $ARGV[0];
17+
if (!defined $stem) {
18+
print STDERR "usage: multi-bleu.pl [-lc] reference < hypothesis\n";
19+
print STDERR "Reads the references from reference or reference0, reference1, ...\n";
20+
exit(1);
21+
}
22+
23+
$stem .= ".ref" if !-e $stem && !-e $stem."0" && -e $stem.".ref0";
24+
25+
my @REF;
26+
my $ref=0;
27+
while(-e "$stem$ref") {
28+
&add_to_ref("$stem$ref",\@REF);
29+
$ref++;
30+
}
31+
&add_to_ref($stem,\@REF) if -e $stem;
32+
die("ERROR: could not find reference file $stem") unless scalar @REF;
33+
34+
# add additional references explicitly specified on the command line
35+
shift;
36+
foreach my $stem (@ARGV) {
37+
&add_to_ref($stem,\@REF) if -e $stem;
38+
}
39+
40+
41+
42+
sub add_to_ref {
43+
my ($file,$REF) = @_;
44+
my $s=0;
45+
if ($file =~ /.gz$/) {
46+
open(REF,"gzip -dc $file|") or die "Can't read $file";
47+
} else {
48+
open(REF,$file) or die "Can't read $file";
49+
}
50+
while(<REF>) {
51+
chop;
52+
push @{$$REF[$s++]}, $_;
53+
}
54+
close(REF);
55+
}
56+
57+
my(@CORRECT,@TOTAL,$length_translation,$length_reference);
58+
my $s=0;
59+
while(<STDIN>) {
60+
chop;
61+
$_ = lc if $lowercase;
62+
my @WORD = split;
63+
my %REF_NGRAM = ();
64+
my $length_translation_this_sentence = scalar(@WORD);
65+
my ($closest_diff,$closest_length) = (9999,9999);
66+
foreach my $reference (@{$REF[$s]}) {
67+
# print "$s $_ <=> $reference\n";
68+
$reference = lc($reference) if $lowercase;
69+
my @WORD = split(' ',$reference);
70+
my $length = scalar(@WORD);
71+
my $diff = abs($length_translation_this_sentence-$length);
72+
if ($diff < $closest_diff) {
73+
$closest_diff = $diff;
74+
$closest_length = $length;
75+
# print STDERR "$s: closest diff ".abs($length_translation_this_sentence-$length)." = abs($length_translation_this_sentence-$length), setting len: $closest_length\n";
76+
} elsif ($diff == $closest_diff) {
77+
$closest_length = $length if $length < $closest_length;
78+
# from two references with the same closeness to me
79+
# take the *shorter* into account, not the "first" one.
80+
}
81+
for(my $n=1;$n<=4;$n++) {
82+
my %REF_NGRAM_N = ();
83+
for(my $start=0;$start<=$#WORD-($n-1);$start++) {
84+
my $ngram = "$n";
85+
for(my $w=0;$w<$n;$w++) {
86+
$ngram .= " ".$WORD[$start+$w];
87+
}
88+
$REF_NGRAM_N{$ngram}++;
89+
}
90+
foreach my $ngram (keys %REF_NGRAM_N) {
91+
if (!defined($REF_NGRAM{$ngram}) ||
92+
$REF_NGRAM{$ngram} < $REF_NGRAM_N{$ngram}) {
93+
$REF_NGRAM{$ngram} = $REF_NGRAM_N{$ngram};
94+
# print "$i: REF_NGRAM{$ngram} = $REF_NGRAM{$ngram}<BR>\n";
95+
}
96+
}
97+
}
98+
}
99+
$length_translation += $length_translation_this_sentence;
100+
$length_reference += $closest_length;
101+
for(my $n=1;$n<=4;$n++) {
102+
my %T_NGRAM = ();
103+
for(my $start=0;$start<=$#WORD-($n-1);$start++) {
104+
my $ngram = "$n";
105+
for(my $w=0;$w<$n;$w++) {
106+
$ngram .= " ".$WORD[$start+$w];
107+
}
108+
$T_NGRAM{$ngram}++;
109+
}
110+
foreach my $ngram (keys %T_NGRAM) {
111+
$ngram =~ /^(\d+) /;
112+
my $n = $1;
113+
# my $corr = 0;
114+
# print "$i e $ngram $T_NGRAM{$ngram}<BR>\n";
115+
$TOTAL[$n] += $T_NGRAM{$ngram};
116+
if (defined($REF_NGRAM{$ngram})) {
117+
if ($REF_NGRAM{$ngram} >= $T_NGRAM{$ngram}) {
118+
$CORRECT[$n] += $T_NGRAM{$ngram};
119+
# $corr = $T_NGRAM{$ngram};
120+
# print "$i e correct1 $T_NGRAM{$ngram}<BR>\n";
121+
}
122+
else {
123+
$CORRECT[$n] += $REF_NGRAM{$ngram};
124+
# $corr = $REF_NGRAM{$ngram};
125+
# print "$i e correct2 $REF_NGRAM{$ngram}<BR>\n";
126+
}
127+
}
128+
# $REF_NGRAM{$ngram} = 0 if !defined $REF_NGRAM{$ngram};
129+
# print STDERR "$ngram: {$s, $REF_NGRAM{$ngram}, $T_NGRAM{$ngram}, $corr}\n"
130+
}
131+
}
132+
$s++;
133+
}
134+
my $brevity_penalty = 1;
135+
my $bleu = 0;
136+
137+
my @bleu=();
138+
139+
for(my $n=1;$n<=4;$n++) {
140+
if (defined ($TOTAL[$n])){
141+
$bleu[$n]=($TOTAL[$n])?$CORRECT[$n]/$TOTAL[$n]:0;
142+
# print STDERR "CORRECT[$n]:$CORRECT[$n] TOTAL[$n]:$TOTAL[$n]\n";
143+
}else{
144+
$bleu[$n]=0;
145+
}
146+
}
147+
148+
if ($length_reference==0){
149+
printf "BLEU = 0, 0/0/0/0 (BP=0, ratio=0, hyp_len=0, ref_len=0)\n";
150+
exit(1);
151+
}
152+
153+
if ($length_translation<$length_reference) {
154+
$brevity_penalty = exp(1-$length_reference/$length_translation);
155+
}
156+
$bleu = $brevity_penalty * exp((my_log( $bleu[1] ) +
157+
my_log( $bleu[2] ) +
158+
my_log( $bleu[3] ) +
159+
my_log( $bleu[4] ) ) / 4) ;
160+
printf "BLEU = %.2f, %.1f/%.1f/%.1f/%.1f (BP=%.3f, ratio=%.3f, hyp_len=%d, ref_len=%d)\n",
161+
100*$bleu,
162+
100*$bleu[1],
163+
100*$bleu[2],
164+
100*$bleu[3],
165+
100*$bleu[4],
166+
$brevity_penalty,
167+
$length_translation / $length_reference,
168+
$length_translation,
169+
$length_reference;
170+
171+
sub my_log {
172+
return -9999999999 unless $_[0];
173+
return log($_[0]);
174+
}

‎scripts/translate/f2e_decode.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
#PBS -q isi
3+
#PBS -l walltime=1:00:00
4+
#PBS -l gpus=2
5+
6+
ROOT=../../ # change your ROOT to some absolute path
7+
8+
EXEC=$ROOT/executable/ZOPH_RNN_XING
9+
10+
# script
11+
12+
PY_FORMAT=$ROOT/scripts/bleu_format_valid.py
13+
PERL_BLEU=$ROOT/scripts/multi-bleu.perl
14+
15+
# model
16+
17+
model_dir=$ROOT/scripts/translate/Fre_Eng_lc_d_att/
18+
fe_nn_attention=$model_dir/best.nn
19+
20+
# data
21+
22+
data_folder=$ROOT/sample_data/
23+
24+
TGT_TST=$data_folder/test_english.tok.lc
25+
SRC_TST=$data_folder/test_french.tok.lc
26+
27+
EN_DEV=$data_folder/dev_english.txt.tok.lc
28+
FR_DEV=$data_folder/dev_french.txt.tok.lc
29+
30+
EN_TRN=$data_folder/train_english.txt.tok.lc.10k
31+
FR_TRN=$data_folder/train_french.txt.tok.lc.10k
32+
33+
# output
34+
35+
output_folder=$ROOT/scripts/translate/Fre_Eng_lc_d_att/decode
36+
mkdir $output_folder
37+
38+
id=FE
39+
LOG=$output_folder/${id}.log
40+
OUTPUT=$output_folder/${id}.kbest
41+
REF=$output_folder/${id}.ref
42+
BLEU=$output_folder/${id}.bleu
43+
44+
# decode
45+
46+
cd $output_folder
47+
48+
$EXEC --decode-main-data-files $SRC_TST -b 12 -L 100 -k 1 $fe_nn_attention $OUTPUT > $LOG
49+
50+
# calculate BLEU
51+
52+
python $PY_FORMAT $OUTPUT $TGT_TST $REF
53+
perl $PERL_BLEU -lc $REF < $OUTPUT.bleu > $BLEU

‎scripts/translate/f2e_train.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
#PBS -q isi80
3+
#PBS -l walltime=300:00:00
4+
#PBS -l gpus=4
5+
6+
ROOT=../../ # change your ROOT to some absolute path
7+
8+
EXEC=$ROOT/executable/ZOPH_RNN_XING
9+
10+
# script
11+
12+
PY_FORMAT=$ROOT/scripts/bleu_format_valid.py
13+
PERL_BLEU=$ROOT/scripts/multi-bleu.perl
14+
15+
# model
16+
17+
model_dir=$ROOT/scripts/translate/Fre_Eng_lc_d_att/
18+
mkdir $model_dir
19+
fe_nn_attention=$model_dir/best.nn
20+
21+
# data
22+
23+
data_folder=$ROOT/sample_data/
24+
25+
TGT_TST=$data_folder/test_english.tok.lc
26+
SRC_TST=$data_folder/test_french.tok.lc
27+
28+
EN_DEV=$data_folder/dev_english.txt.tok.lc
29+
FR_DEV=$data_folder/dev_french.txt.tok.lc
30+
31+
EN_TRN=$data_folder/train_english.txt.tok.lc.10k
32+
FR_TRN=$data_folder/train_french.txt.tok.lc.10k
33+
34+
# train
35+
36+
cd $model_dir
37+
38+
$EXEC --logfile HPC_OUTPUT_NEW.txt -B best.nn --screen-print-rate 300 -N 2 -M 0 1 2 --attention-model true --feed-input true -H 1000 -a $FR_DEV $EN_DEV -t $FR_TRN $EN_TRN model.nn -v 200000 -V 40000 -L 100 -n 8 -A 1 --fixed-halve-lr 6 -l 0.35 -d 0.8 -m 128 -w 5

0 commit comments

Comments
 (0)