scratch/KatzTrigramLanguageModel.java at master · debanjanghosh/scratch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
package nlp.assignments.lm;


import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import nlp.langmodel.LanguageModel;
import nlp.util.Counter;
import nlp.util.CounterMap;

/**
 * A dummy language model -- uses empirical unigram counts, plus a single
 * ficticious count for unknown words.
 */
public class KatzTrigramLanguageModel implements LanguageModel {

  static final String START = "<S>";
  static final String STOP = "</S>";
  static final String UNKNOWN = "*UNKNOWN*";
  static final double lambda1 = 0.45;
  static final double lambda2 = 0.5;

  Counter<String> wordCounter = new Counter<String>();
  CounterMap<String, String> bigramCounter = new CounterMap<String, String>();
  CounterMap<String, String> trigramCounter = new CounterMap<String, String>();

  public double getTrigramProbability(String prePreviousWord, String previousWord, String word) {
	double trigramCount = trigramCounter.getCount(prePreviousWord+previousWord, word);
	double bigramCount = bigramCounter.getCount(previousWord, word);
    double unigramCount = wordCounter.getCount(word);
    if (unigramCount == 0) {
      System.out.println("UNKNOWN Word: " + word);
      unigramCount = wordCounter.getCount(UNKNOWN);
    }
    return lambda1*trigramCount + lambda2*bigramCount + (1.0-lambda1-lambda2)*unigramCount;
  }

  public double getSentenceProbability(List<String> sentence) {
    List<String> stoppedSentence = new ArrayList<String>(sentence);
    stoppedSentence.add(0, START);
    stoppedSentence.add(0, START);
    stoppedSentence.add(STOP);
    double probability = 1.0;
    String prePreviousWord = stoppedSentence.get(0);
    String previousWord = stoppedSentence.get(1);
    for (int i = 2; i < stoppedSentence.size(); i++) {
      String word = stoppedSentence.get(i);
      probability *= getTrigramProbability(prePreviousWord, previousWord, word);
      prePreviousWord = previousWord;
      previousWord = word;
    }
    if (probability == 0) System.err.println("Underflow");
    return probability;
  }

  String generateWord() {
    double sample = Math.random();
    double sum = 0.0;
    for (String word : wordCounter.keySet()) {
      sum += wordCounter.getCount(word);
      if (sum > sample) {
        return word;
      }
    }
    return UNKNOWN;
  }

  public List<String> generateSentence() {
    List<String> sentence = new ArrayList<String>();
    String word = generateWord();
    while (!word.equals(STOP)) {
      sentence.add(word);
      word = generateWord();
    }
    return sentence;
  }

  public KatzTrigramLanguageModel(Collection<List<String>> sentenceCollection) {
    for (List<String> sentence : sentenceCollection) {
      List<String> stoppedSentence = new ArrayList<String>(sentence);
      stoppedSentence.add(0, START);
      stoppedSentence.add(0, START);
      stoppedSentence.add(STOP);
      String prePreviousWord = stoppedSentence.get(0);
      String previousWord = stoppedSentence.get(1);
      for (int i = 2; i < stoppedSentence.size(); i++) {
    	String word = stoppedSentence.get(i);
        wordCounter.incrementCount(word, 1.0);
        bigramCounter.incrementCount(previousWord, word, 1.0);
        trigramCounter.incrementCount(prePreviousWord+previousWord, word, 1.0);
        prePreviousWord = previousWord;
        previousWord = word;
      }
    }
    wordCounter.incrementCount(UNKNOWN, 1.0);
    normalizeDistributions();
  }

  private void normalizeDistributions() {
	 for (String previousBigram : trigramCounter.keySet()) {
		 trigramCounter.getCounter(previousBigram).normalize();
	 }
	 for (String previousWord : bigramCounter.keySet()) {
		 bigramCounter.getCounter(previousWord).normalize();
	 }
	 wordCounter.normalize();
  }
}