Commit ·
117e3f3
1
Parent(s): ccc04ba
Upload 2 files
Browse files- ChatIPC.cpp +6 -7
ChatIPC.cpp
CHANGED
|
@@ -507,7 +507,7 @@ static std::vector<std::string> digit_to_words(const std::string& s) {
|
|
| 507 |
}
|
| 508 |
std::string num = s.substr(first_non_zero);
|
| 509 |
|
| 510 |
-
// If it's
|
| 511 |
// and maintain semantic definitions of the numbers.
|
| 512 |
if (num.length() > 69) {
|
| 513 |
for (char c : num) {
|
|
@@ -1151,7 +1151,7 @@ static std::string best_candidate_by_similarity(
|
|
| 1151 |
// Add the sub-token itself to the bitset
|
| 1152 |
bitset_set(agg_words.data(), words, sub_tid);
|
| 1153 |
|
| 1154 |
-
// Retrieve the interned pointer
|
| 1155 |
StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
|
| 1156 |
if (sub_ptr) {
|
| 1157 |
auto sub_it = def_index.find(sub_ptr);
|
|
@@ -1197,14 +1197,14 @@ static std::string best_candidate_by_similarity(
|
|
| 1197 |
std::vector<std::string> sub_tokens = tokenize_others(*cand);
|
| 1198 |
for (const std::string& sub_tok : sub_tokens) {
|
| 1199 |
|
| 1200 |
-
// Check if the sub-token exists in
|
| 1201 |
TokenId sub_tid = interner.id_of(sub_tok);
|
| 1202 |
|
| 1203 |
if (sub_tid != TOKEN_ID_INVALID) {
|
| 1204 |
// Add the sub-token itself to the candidate's semantic fingerprint (row)
|
| 1205 |
bitset_set(row, words, sub_tid);
|
| 1206 |
|
| 1207 |
-
// Convert the ID back to a pointer
|
| 1208 |
StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
|
| 1209 |
if (sub_ptr) {
|
| 1210 |
auto sub_it = def_index.find(sub_ptr);
|
|
@@ -1655,7 +1655,7 @@ static void print_kb_info(const std::string &fname) {
|
|
| 1655 |
return;
|
| 1656 |
}
|
| 1657 |
|
| 1658 |
-
std::cout << "Reckoning knowledge-base " << fname << " info.
|
| 1659 |
|
| 1660 |
try {
|
| 1661 |
const std::uint64_t magic = read_u64(ifs);
|
|
@@ -1700,8 +1700,7 @@ static void print_kb_info(const std::string &fname) {
|
|
| 1700 |
}
|
| 1701 |
}
|
| 1702 |
|
| 1703 |
-
std::cout << fname << " dictionary depth
|
| 1704 |
-
std::cout << fname << " n-gram max size: " << max_ngram << "\n";
|
| 1705 |
|
| 1706 |
} catch (const std::exception &e) {
|
| 1707 |
std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
|
|
|
|
| 507 |
}
|
| 508 |
std::string num = s.substr(first_non_zero);
|
| 509 |
|
| 510 |
+
// If it's very large (e.g. googolplex logic), fall back to single digits to prevent overflow
|
| 511 |
// and maintain semantic definitions of the numbers.
|
| 512 |
if (num.length() > 69) {
|
| 513 |
for (char c : num) {
|
|
|
|
| 1151 |
// Add the sub-token itself to the bitset
|
| 1152 |
bitset_set(agg_words.data(), words, sub_tid);
|
| 1153 |
|
| 1154 |
+
// Retrieve the interned pointer and look it up in def_index
|
| 1155 |
StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
|
| 1156 |
if (sub_ptr) {
|
| 1157 |
auto sub_it = def_index.find(sub_ptr);
|
|
|
|
| 1197 |
std::vector<std::string> sub_tokens = tokenize_others(*cand);
|
| 1198 |
for (const std::string& sub_tok : sub_tokens) {
|
| 1199 |
|
| 1200 |
+
// Check if the sub-token exists in the knowledge base
|
| 1201 |
TokenId sub_tid = interner.id_of(sub_tok);
|
| 1202 |
|
| 1203 |
if (sub_tid != TOKEN_ID_INVALID) {
|
| 1204 |
// Add the sub-token itself to the candidate's semantic fingerprint (row)
|
| 1205 |
bitset_set(row, words, sub_tid);
|
| 1206 |
|
| 1207 |
+
// Convert the ID back to a pointer
|
| 1208 |
StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
|
| 1209 |
if (sub_ptr) {
|
| 1210 |
auto sub_it = def_index.find(sub_ptr);
|
|
|
|
| 1655 |
return;
|
| 1656 |
}
|
| 1657 |
|
| 1658 |
+
std::cout << "Reckoning knowledge-base " << fname << " info. ";
|
| 1659 |
|
| 1660 |
try {
|
| 1661 |
const std::uint64_t magic = read_u64(ifs);
|
|
|
|
| 1700 |
}
|
| 1701 |
}
|
| 1702 |
|
| 1703 |
+
std::cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
|
|
|
|
| 1704 |
|
| 1705 |
} catch (const std::exception &e) {
|
| 1706 |
std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
|