-
Notifications
You must be signed in to change notification settings - Fork 57
/
Copy pathtests.py
151 lines (111 loc) · 4.65 KB
/
tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
from __future__ import print_function
from unittest import TestCase
from utils import unsort
import unittest
import accuracy
import numpy as np
import text_manipulation
class LoaderTests(TestCase):
def testReallyTrivial(self):
assert 1 + 1 == 2
class PkTests(unittest.TestCase):
def test_get_boundaries(self):
sentences_class = []
sentences_class.append(("first sen.", 1))
sentences_class.append(("sec sen.", 1))
sentences_class.append(("third sen.", 0))
sentences_class.append(("forth sen.", 1))
sentences_class.append(("fifth sen.", 0))
sentences_class.append(("sixth sen.", 0))
sentences_class.append(("seventh sen.", 1))
expected = [2, 2, 4, 6]
result = accuracy.get_seg_boundaries(sentences_class)
for i, num in enumerate(result):
self.assertTrue(num == expected[i])
def test_get_boundaries2(self):
sentences_class = []
sentences_class.append(("first sen is 5 words.", 0))
sentences_class.append(("sec sen.", 0))
sentences_class.append(("third sen is a very very very long sentence.", 1))
sentences_class.append(("the forth one is single segment.", 1))
expected = [16, 6]
result = accuracy.get_seg_boundaries(sentences_class)
for i, num in enumerate(result):
self.assertTrue(num == expected[i])
def test_pk_perefct_seg(self):
sentences_class = []
sentences_class.append(("first sen is 5 words.", 0))
sentences_class.append(("sec sen.", 0))
sentences_class.append(("third sen is a very very very long sentence.", 1))
sentences_class.append(("the forth one is single segment.", 1))
gold = accuracy.get_seg_boundaries(sentences_class)
h = accuracy.get_seg_boundaries(sentences_class)
# with specified window size
for window_size in range(1, 15, 1):
acc = accuracy.pk(gold, h, window_size=window_size)
self.assertEquals(acc, 1)
# with default window size
acc = accuracy.pk(gold, h)
self.assertEquals(acc, 1)
def test_pk_false_neg(self):
h = []
h.append(("5 words sentence of data.", 0))
h.append(("2 sentences same seg.", 1))
gold = []
gold.append(("5 words sentence of data.", 1))
gold.append(("2 sentences same seg.", 1))
gold = accuracy.get_seg_boundaries(gold)
h = accuracy.get_seg_boundaries(h)
window_size = 3
comparison_count = 6
# with default window size
acc = accuracy.pk(gold, h)
self.assertEquals(acc, window_size / comparison_count)
window_size = 4
acc = accuracy.pk(gold, h)
self.assertEquals(acc, window_size / comparison_count)
def test_windiff(self):
h = []
h.append(("5 words sentence of data.", 0))
h.append(("short.", 1))
h.append(("extra segmented sen.", 1))
h.append(("last and very very very very very long sen.", 1))
gold = []
gold.append(("5 words sentence of data.", 1))
gold.append(("short.", 1))
gold.append(("extra segmented sen.", 0))
gold.append(("last and very very very very very long sen.", 1))
gold = accuracy.get_seg_boundaries(gold)
h = accuracy.get_seg_boundaries(h)
window_size = 3
acc = accuracy.win_diff(gold, h, window_size = window_size)
self.assertEquals(float(acc), 0.6)
window_size = 5
expected = float(1)- float(8) / 13
acc = accuracy.win_diff(gold, h, window_size=window_size)
self.assertEquals("{0:.5f}".format(float(acc)), "{0:.5f}".format(expected))
class UnsortTests(TestCase):
def test_unsort(self):
x = np.random.randint(0, 100, 10)
sort_order = np.argsort(x)
unsort_order = unsort(sort_order)
assert np.all(x[sort_order][unsort_order] == x)
class SentenceTokenizerTests(TestCase):
def test_a_little(self):
a = text_manipulation.split_sentences(u"Hello, Mr. Trump, how do you do? What? Where? I don't i.e e.g Russia.")
assert a == [u'Hello, Mr. Trump, how do you do?',
u'What?',
u'Where?',
u"I don't i.e e.g Russia."]
def test_linebreaks(self):
text = u'''Line one. Still line one.
Line two. Can I span
two lines?'''
a = text_manipulation.split_sentences(text)
print(a)
assert a == [u'Line one.',
u'Still line one.',
u'Line two.',
u'Can I span\n two lines?']
if __name__ == '__main__':
unittest.main()