technician1 commited on
Commit
117e3f3
·
1 Parent(s): ccc04ba

Upload 2 files

Browse files
Files changed (1) hide show
  1. ChatIPC.cpp +6 -7
ChatIPC.cpp CHANGED
@@ -507,7 +507,7 @@ static std::vector<std::string> digit_to_words(const std::string& s) {
507
  }
508
  std::string num = s.substr(first_non_zero);
509
 
510
- // If it's absurdly large (e.g. googolplex logic), fall back to single digits to prevent overflow
511
  // and maintain semantic definitions of the numbers.
512
  if (num.length() > 69) {
513
  for (char c : num) {
@@ -1151,7 +1151,7 @@ static std::string best_candidate_by_similarity(
1151
  // Add the sub-token itself to the bitset
1152
  bitset_set(agg_words.data(), words, sub_tid);
1153
 
1154
- // Retrieve the interned pointer so we can look it up in def_index
1155
  StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
1156
  if (sub_ptr) {
1157
  auto sub_it = def_index.find(sub_ptr);
@@ -1197,14 +1197,14 @@ static std::string best_candidate_by_similarity(
1197
  std::vector<std::string> sub_tokens = tokenize_others(*cand);
1198
  for (const std::string& sub_tok : sub_tokens) {
1199
 
1200
- // Check if the sub-token exists in our knowledge base
1201
  TokenId sub_tid = interner.id_of(sub_tok);
1202
 
1203
  if (sub_tid != TOKEN_ID_INVALID) {
1204
  // Add the sub-token itself to the candidate's semantic fingerprint (row)
1205
  bitset_set(row, words, sub_tid);
1206
 
1207
- // Convert the ID back to a pointer so we can check the dictionary
1208
  StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
1209
  if (sub_ptr) {
1210
  auto sub_it = def_index.find(sub_ptr);
@@ -1655,7 +1655,7 @@ static void print_kb_info(const std::string &fname) {
1655
  return;
1656
  }
1657
 
1658
- std::cout << "Reckoning knowledge-base " << fname << " info.\n";
1659
 
1660
  try {
1661
  const std::uint64_t magic = read_u64(ifs);
@@ -1700,8 +1700,7 @@ static void print_kb_info(const std::string &fname) {
1700
  }
1701
  }
1702
 
1703
- std::cout << fname << " dictionary depth: " << file_def_depth << "\n";
1704
- std::cout << fname << " n-gram max size: " << max_ngram << "\n";
1705
 
1706
  } catch (const std::exception &e) {
1707
  std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
 
507
  }
508
  std::string num = s.substr(first_non_zero);
509
 
510
+ // If it's very large (e.g. googolplex logic), fall back to single digits to prevent overflow
511
  // and maintain semantic definitions of the numbers.
512
  if (num.length() > 69) {
513
  for (char c : num) {
 
1151
  // Add the sub-token itself to the bitset
1152
  bitset_set(agg_words.data(), words, sub_tid);
1153
 
1154
+ // Retrieve the interned pointer and look it up in def_index
1155
  StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
1156
  if (sub_ptr) {
1157
  auto sub_it = def_index.find(sub_ptr);
 
1197
  std::vector<std::string> sub_tokens = tokenize_others(*cand);
1198
  for (const std::string& sub_tok : sub_tokens) {
1199
 
1200
+ // Check if the sub-token exists in the knowledge base
1201
  TokenId sub_tid = interner.id_of(sub_tok);
1202
 
1203
  if (sub_tid != TOKEN_ID_INVALID) {
1204
  // Add the sub-token itself to the candidate's semantic fingerprint (row)
1205
  bitset_set(row, words, sub_tid);
1206
 
1207
+ // Convert the ID back to a pointer
1208
  StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
1209
  if (sub_ptr) {
1210
  auto sub_it = def_index.find(sub_ptr);
 
1655
  return;
1656
  }
1657
 
1658
+ std::cout << "Reckoning knowledge-base " << fname << " info. ";
1659
 
1660
  try {
1661
  const std::uint64_t magic = read_u64(ifs);
 
1700
  }
1701
  }
1702
 
1703
+ std::cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
 
1704
 
1705
  } catch (const std::exception &e) {
1706
  std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";