deepdive/examples/chunking/app.ddlog at master · Behrad3d/deepdive · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
tags(
  tag text,
  tag_id int
).

words_raw(
  sent_id bigint,
  word_id bigint,
  word text,
  pos text,
  true_tag text
).

words(
  sent_id bigint,
  word_id bigint,
  word text,
  pos text,
  true_tag text,
  tag int
).

word_features(
  word_id bigint,
  feature text
).

words(sent_id, word_id, word, pos, true_tag, tag_id) :-
  words_raw(sent_id, word_id, word, pos, true_tag),
  tags(tag, tag_id),
  if true_tag = "B-UCP" then ""
  else if true_tag = "I-UCP" then ""
  else if strpos(true_tag, "-") > 0 then
     split_part(true_tag, "-", 2)
  else if true_tag = "O" then "O"
  else ""
  end = tag.

function ext_features
  over (word_id1 bigint, word1 text, pos1 text, word2 text, pos2 text)
  returns rows like word_features
  implementation "udf/ext_features.py" handles tsv lines.

word_features +=
  ext_features(word_id1, word1, pos1, word2, pos2) :-
  words(sent_id, word_id1, word1, pos1, _, _),
  words(sent_id, word_id2, word2, pos2, _, _),
  [word_id1 = word_id2 + 1],
  word1 IS NOT NULL.


tag?(word_id bigint) Categorical(13).

@label(tag)
tag(word_id) :- words(_, word_id, _, _, _, tag).

@weight(f)
tag(word_id) :- word_features(word_id, f).

@weight("?")
Multinomial(tag(word_id_1), tag(word_id_2)) :-
  words(_, word_id_1, _, _, _, _),
  words(_, word_id_2, _, _, _, _),
  word_id_2=word_id_1+1.

@weight("?")
Multinomial(tag(word_id_1), tag(word_id_2)) :-
  words(sent_id, word_id_1, word, _, _, tag),
  words(sent_id, word_id_2, word, _, _, _),
  tag IS NOT NULL,
  word_id_1<word_id_2.