-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfasttextRun.py
executable file
·58 lines (51 loc) · 1.71 KB
/
fasttextRun.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#!/usr/bin/python3
"""
fasttextRun.py: run fasttext via python interface
usage: fasttextRun.py -f file [-n N]
note: default number of N is 10 (10-fold cross validation)
20180105 erikt(at)xs4all.nl
"""
import fasttext
import os
import random
import splitFile
import sys
COMMAND = sys.argv.pop(0)
DIM = 300
LARGENUMBER = 100000
MINCOUNT = 5
random.seed()
TMPFILENAME = "fasttextRun."+str(os.getpid())+"."+str(random.randint(0,LARGENUMBER))
def makeTrainFile(inFileName,i,n):
outFileName = TMPFILENAME+".train"
outFile = open(outFileName,"w")
for j in range(0,n):
if j != i:
inFile = open(inFileName+"."+str(j),"r")
for line in inFile: outFile.write(line)
inFile.close()
outFile.close()
return(outFileName)
def fasttextRun(inFileName,i,n):
trainFileName = makeTrainFile(inFileName,i,n)
modelFileName = TMPFILENAME+".model"
testFileName = inFileName+"."+str(i)
classifier = fasttext.supervised(trainFileName,modelFileName,dim=DIM,min_count=MINCOUNT)
# ,pretrained_vectors="/home/erikt/software/fastText/wiki.nl.vec")
result = classifier.test(testFileName)
os.unlink(trainFileName)
os.unlink(modelFileName+".bin")
return(result.precision)
def main(argv):
inFileName, n = splitFile.processOpts(list(argv))
data = splitFile.readData(inFileName)
splitFile.writeData(inFileName,data,n)
accuracyTotal = 0.0
for i in range(0,n):
accuracy = fasttextRun(inFileName,i,n)
accuracyTotal += accuracy
print("Fold: {0:0d}; Accuracy: {1:0.3f}".format(i,accuracy))
print("Average accuracy {0:0.3f}".format(accuracyTotal/float(n)))
return(0)
if __name__ == "__main__":
sys.exit(main(sys.argv))