40 if (words_[0] !=
NULL) {
60 file_name = data_file_path +
lang;
61 file_name +=
".cube.word-freq";
69 vector<string> str_vec;
71 if (str_vec.size() < 2) {
77 if (word_unigrams_obj ==
NULL) {
78 fprintf(stderr,
"Cube ERROR (WordUnigrams::Create): could not create "
79 "word unigrams object.\n");
83 int full_len = str.length();
84 int word_cnt = str_vec.size() / 2;
85 word_unigrams_obj->words_ =
new char*[word_cnt];
86 word_unigrams_obj->costs_ =
new int[word_cnt];
88 if (word_unigrams_obj->words_ ==
NULL ||
89 word_unigrams_obj->costs_ ==
NULL) {
90 fprintf(stderr,
"Cube ERROR (WordUnigrams::Create): error allocating "
91 "word unigram fields.\n");
92 delete word_unigrams_obj;
96 word_unigrams_obj->words_[0] =
new char[full_len];
97 if (word_unigrams_obj->words_[0] ==
NULL) {
98 fprintf(stderr,
"Cube ERROR (WordUnigrams::Create): error allocating "
99 "word unigram fields.\n");
100 delete word_unigrams_obj;
105 word_unigrams_obj->word_cnt_ = 0;
106 char *char_buff = word_unigrams_obj->words_[0];
110 for (
int wrd = 0; wrd < str_vec.size(); wrd += 2) {
111 word_unigrams_obj->words_[word_cnt] = char_buff;
113 strcpy(char_buff, str_vec[wrd].c_str());
114 char_buff += (str_vec[wrd].length() + 1);
116 if (sscanf(str_vec[wrd + 1].c_str(),
"%d",
117 word_unigrams_obj->costs_ + word_cnt) != 1) {
118 fprintf(stderr,
"Cube ERROR (WordUnigrams::Create): error reading "
119 "word unigram data.\n");
120 delete word_unigrams_obj;
124 max_cost =
MAX(max_cost, word_unigrams_obj->costs_[word_cnt]);
127 word_unigrams_obj->word_cnt_ = word_cnt;
140 word_unigrams_obj->not_in_list_cost_ = max_cost +
143 return word_unigrams_obj;
158 vector<string> words;
162 if (words.size() <= 0) {
168 for (
int word_idx = 0; word_idx < words.size(); word_idx++) {
178 while (clean_len > 0 &&
189 if (clean_len == 0 || !trunc) {
192 clean_str32 =
new char_32[clean_len + 1];
193 for (
int i = 0; i < clean_len; ++i) {
194 clean_str32[i] = str32[i];
196 clean_str32[clean_len] =
'\0';
206 if (clean_len >= kMinLengthNumOrCaseInvariant &&
224 if (clean_len >= kMinLengthNumOrCaseInvariant) {
226 bool is_numeric =
true;
227 for (
int i = 0; i < clean_len; ++i) {
228 if (!lang_mod->
IsDigit(clean_str32[i]))
234 delete [] clean_str32;
239 return static_cast<int>(cost /
static_cast<double>(words.size()));
244 if (strlen(key_str) == 0)
245 return not_in_list_cost_;
246 int hi = word_cnt_ - 1;
249 int current = (hi + lo) / 2;
250 int comp = strcmp(key_str, words_[current]);
253 return costs_[current];
263 return not_in_list_cost_;
static int Prob2Cost(double prob_val)
static bool ReadFileToString(const string &file_name, string *str)
static char_32 * ToLower(const char_32 *str32, CharSet *char_set)
static void UTF8ToUTF32(const char *utf8_str, string_32 *str32)
static WordUnigrams * Create(const string &data_file_path, const string &lang)
int Cost(const char_32 *str32, LangModel *lang_mod, CharSet *char_set) const
int CostInternal(const char *str) const
static int StrLen(const char_32 *str)
static void UTF32ToUTF8(const char_32 *utf32_str, string *str)
static void SplitStringUsing(const string &str, const string &delims, vector< string > *str_vec)
virtual bool IsDigit(char_32 ch)=0
static char_32 * ToUpper(const char_32 *str32, CharSet *char_set)
virtual bool IsTrailingPunc(char_32 ch)=0
static bool IsCaseInvariant(const char_32 *str32, CharSet *char_set)
basic_string< char_32 > string_32
static char_32 * StrDup(const char_32 *str)