Introduced InnerSegments, InnerSegmentBoundary class

This utility class is introduced to provide more consistent access
to the inner segment information, while hiding the internal encoding schema, allowing more flexible implementation.

- Provides a modern range-based loop-based iterator.
- Provides a builder class for inner boundary to encapsulate the encoding algorithm and error-prone data generation.
- Automatic fallback when invalid or empty boundary information is passed.
- Provides more consistent behavior for content_(key|value) and inner segment boundaries.

PiperOrigin-RevId: 790999040

Author: taku910

src/converter/BUILD.bazel

@@ -58,12 +58,14 @@ mozc_cc_library(
     ],
     deps = [
         ":attribute",
+        ":inner_segment",
         ":lattice",
         "//base:number_util",
         "//base:util",
         "//base:vlog",
         "//base/container:freelist",
         "//base/strings:assign",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/log",
         "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/strings",
@@ -91,6 +93,7 @@ mozc_cc_test(
     size = "small",
     srcs = ["candidate_test.cc"],
     deps = [
+        ":inner_segment",
         ":segments",
         "//base:number_util",
         "//testing:gunit_main",
@@ -108,6 +111,41 @@ mozc_cc_library(
     ],
 )
 
+mozc_cc_library(
+    name = "inner_segment",
+    hdrs = [
+        "inner_segment.h",
+    ],
+    visibility = [
+        "//engine:__pkg__",
+        "//prediction:__pkg__",
+        "//request:__pkg__",
+        "//rewriter:__pkg__",
+    ],
+    deps = [
+        "@com_google_absl//absl/container:fixed_array",
+        "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+mozc_cc_test(
+    name = "inner_segment_test",
+    srcs = [
+        "inner_segment_test.cc",
+    ],
+    deps = [
+        ":inner_segment",
+        "//testing:gunit_main",
+        "@com_google_absl//absl/log",
+        "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
 mozc_cc_library(
     name = "segments_matchers",
     testonly = 1,
@@ -223,6 +261,7 @@ mozc_cc_library(
     deps = [
         ":candidate_filter",
         ":connector",
+        ":inner_segment",
         ":lattice",
         ":node",
         ":segmenter",
@@ -423,6 +462,7 @@ mozc_cc_test(
     ],
     deps = [
         ":immutable_converter_no_factory",
+        ":inner_segment",
         ":lattice",
         ":node",
         ":segments",
@@ -475,6 +515,7 @@ mozc_cc_library(
         ":converter_interface",
         ":history_reconstructor",
         ":immutable_converter_interface",
+        ":inner_segment",
         ":reverse_converter",
         ":segments",
         "//base:util",
@@ -509,6 +550,7 @@ mozc_cc_test(
         ":converter_interface",
         ":immutable_converter_interface",
         ":immutable_converter_no_factory",
+        ":inner_segment",
         ":segments",
         ":segments_matchers",
         "//base:util",

src/converter/candidate.cc

@@ -44,6 +44,7 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/span.h"
 #include "base/number_util.h"
+#include "converter/inner_segment.h"
 
 #ifdef MOZC_CANDIDATE_DEBUG
 #include "absl/strings/str_cat.h"
@@ -85,54 +86,6 @@ void Candidate::Dlog(absl::string_view filename, int line,
 }
 #endif  // MOZC_CANDIDATE_DEBUG
 
-bool Candidate::IsValid() const {
-  if (inner_segment_boundary.empty()) {
-    return true;
-  }
-  // The sums of the lengths of key, value components must coincide with those
-  // of key, value, respectively.
-  size_t sum_key_len = 0, sum_value_len = 0;
-  for (InnerSegmentIterator iter(this); !iter.Done(); iter.Next()) {
-    sum_key_len += iter.GetKey().size();
-    sum_value_len += iter.GetValue().size();
-  }
-  return sum_key_len == key.size() && sum_value_len == value.size();
-}
-
-bool Candidate::EncodeLengths(size_t key_len, size_t value_len,
-                              size_t content_key_len, size_t content_value_len,
-                              uint32_t *result) {
-  if (key_len > std::numeric_limits<uint8_t>::max() ||
-      value_len > std::numeric_limits<uint8_t>::max() ||
-      content_key_len > std::numeric_limits<uint8_t>::max() ||
-      content_value_len > std::numeric_limits<uint8_t>::max()) {
-    return false;
-  }
-  *result = (static_cast<uint32_t>(key_len) << 24) |
-            (static_cast<uint32_t>(value_len) << 16) |
-            (static_cast<uint32_t>(content_key_len) << 8) |
-            static_cast<uint32_t>(content_value_len);
-  return true;
-}
-
-std::tuple<size_t, size_t, size_t, size_t> Candidate::DecodeLengths(
-    uint32_t encoded) {
-  return std::make_tuple(encoded >> 24, (encoded >> 16) & 0xff,
-                         (encoded >> 8) & 0xff, (encoded & 0xff));
-}
-
-bool Candidate::PushBackInnerSegmentBoundary(size_t key_len, size_t value_len,
-                                             size_t content_key_len,
-                                             size_t content_value_len) {
-  uint32_t encoded;
-  if (EncodeLengths(key_len, value_len, content_key_len, content_value_len,
-                    &encoded)) {
-    inner_segment_boundary.push_back(encoded);
-    return true;
-  }
-  return false;
-}
-
 std::string Candidate::DebugString() const {
   std::stringstream os;
   os << "(key=" << key << " ckey=" << content_key << " val=" << value
@@ -151,70 +104,15 @@ std::string Candidate::DebugString() const {
   }
   if (!inner_segment_boundary.empty()) {
     os << " segbdd=";
-    for (size_t i = 0; i < inner_segment_boundary.size(); ++i) {
-      const uint32_t encoded_lengths = inner_segment_boundary[i];
-      const auto [key_len, value_len, content_key_len, content_value_len] =
-          DecodeLengths(encoded_lengths);
-      os << absl::StreamFormat("<%d,%d,%d,%d>", key_len, value_len,
-                               content_key_len, content_value_len);
+    for (const auto &iter : inner_segments()) {
+      os << absl::StreamFormat(
+          "<%d,%d,%d,%d>", iter.GetKey().size(), iter.GetValue().size(),
+          iter.GetContentKey().size(), iter.GetContentValue().size());
     }
   }
   os << ")" << std::endl;
   return os.str();
 }
 
-void Candidate::InnerSegmentIterator::Next() {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_++];
-  key_offset_ += encoded_lengths >> 24;
-  value_offset_ += (encoded_lengths >> 16) & 0xff;
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetKey() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(key_offset_, encoded_lengths >> 24);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetValue() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(value_offset_, (encoded_lengths >> 16) & 0xff);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetContentKey() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(key_offset_, (encoded_lengths >> 8) & 0xff);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetContentValue() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(value_offset_, encoded_lengths & 0xff);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetFunctionalKey() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  const int key_len = encoded_lengths >> 24;
-  const int content_key_len = (encoded_lengths >> 8) & 0xff;
-  if (const int key_size = key_len - content_key_len; key_size > 0) {
-    return absl::string_view(key_offset_ + content_key_len, key_size);
-  }
-  return absl::string_view();
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetFunctionalValue() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  const int value_len = (encoded_lengths >> 16) & 0xff;
-  const int content_value_len = encoded_lengths & 0xff;
-  if (const int value_size = value_len - content_value_len; value_size > 0) {
-    return absl::string_view(value_offset_ + content_value_len, value_size);
-  }
-  return absl::string_view();
-}
-
 }  // namespace converter
 }  // namespace mozc

src/converter/candidate.h

@@ -37,10 +37,12 @@
 #include <tuple>
 #include <vector>
 
+#include "absl/base/attributes.h"
 #include "absl/strings/string_view.h"
 #include "absl/types/span.h"
 #include "base/number_util.h"
 #include "converter/attribute.h"
+#include "converter/inner_segment.h"
 
 #ifndef NDEBUG
 #define MOZC_CANDIDATE_DEBUG
@@ -137,7 +139,7 @@ class Candidate {
   // Boundary information for real time conversion.  This will be set only for
   // real time conversion result candidates.  Each element is the encoded
   // lengths of key, value, content key and content value.
-  std::vector<uint32_t> inner_segment_boundary;
+  InnerSegmentBoundary inner_segment_boundary;
   // LINT.ThenChange(//converter/segments_matchers.h)
 
   // The original cost before rescoring. Used for debugging purpose.
@@ -148,68 +150,11 @@ class Candidate {
   mutable std::string log;
 #endif  // MOZC_CANDIDATE_DEBUG
 
-  static bool EncodeLengths(size_t key_len, size_t value_len,
-                            size_t content_key_len, size_t content_value_len,
-                            uint32_t *result);
-
-  // This function ignores error, so be careful when using this.
-  static uint32_t EncodeLengths(size_t key_len, size_t value_len,
-                                size_t content_key_len,
-                                size_t content_value_len) {
-    uint32_t result;
-    EncodeLengths(key_len, value_len, content_key_len, content_value_len,
-                  &result);
-    return result;
+  InnerSegments inner_segments() const {
+    return InnerSegments(key, value, content_key, content_value,
+                         inner_segment_boundary);
   }
 
-  // returns [key_len, value_len, content_value_len, content_value_len]
-  static std::tuple<size_t, size_t, size_t, size_t> DecodeLengths(
-      uint32_t encoded);
-
-  // Inserts a new element to |inner_segment_boundary|.  If one of four
-  // lengths is longer than 255, this method returns false.
-  bool PushBackInnerSegmentBoundary(size_t key_len, size_t value_len,
-                                    size_t content_key_len,
-                                    size_t content_value_len);
-
-  // Iterates inner segments.  Usage example:
-  // for (InnerSegmentIterator iter(&cand); !iter.Done(); iter.Next()) {
-  //   absl::string_view s = iter.GetContentKey();
-  //   ...
-  // }
-  class InnerSegmentIterator final {
-   public:
-    explicit InnerSegmentIterator(const Candidate *candidate)
-        : inner_segment_boundary_(candidate->inner_segment_boundary),
-          key_offset_(candidate->key.data()),
-          value_offset_(candidate->value.data()),
-          index_(0) {}
-
-    InnerSegmentIterator(absl::Span<const uint32_t> inner_segment_boundary,
-                         absl::string_view key, absl::string_view value)
-        : inner_segment_boundary_(inner_segment_boundary),
-          key_offset_(key.data()),
-          value_offset_(value.data()),
-          index_(0) {}
-
-    bool Done() const { return index_ == inner_segment_boundary_.size(); }
-
-    void Next();
-    absl::string_view GetKey() const;
-    absl::string_view GetValue() const;
-    absl::string_view GetContentKey() const;
-    absl::string_view GetContentValue() const;
-    absl::string_view GetFunctionalKey() const;
-    absl::string_view GetFunctionalValue() const;
-    size_t GetIndex() const { return index_; }
-
-   private:
-    const absl::Span<const uint32_t> inner_segment_boundary_;
-    const char *key_offset_ = nullptr;
-    const char *value_offset_ = nullptr;
-    size_t index_ = 0;
-  };
-
   // Clears the Candidate with default values. Note that the default
   // constructor already does the same so you don't need to call Clear
   // explicitly.
@@ -225,18 +170,11 @@ class Candidate {
   // value.substr(content_value.size(), value.size() - content_value.size());
   absl::string_view functional_value() const;
 
-  // Returns whether the inner_segment_boundary member is consistent with
-  // key and value.
-  // Note: content_key and content_value are not checked here.
-  // We cannot compose candidate's content_key and content_value directly
-  // from the inner segments in the current implementation.
-  // Example:
-  // value: 車のほうがあとだ
-  // content_value: 車のほうがあとだ
-  // inner_segments:
-  // <くるまのほうが, 車のほうが, くるま, 車>
-  // <あとだ, あとだ, あとだ, あとだ>
-  bool IsValid() const;
+  // inner_segments() always returns valid information, so
+  // IsValid() can return always true.
+  // TODO(taku): Remove this method.
+  ABSL_DEPRECATED("IsValid() always returns true.")
+  bool IsValid() const { return true; }
   std::string DebugString() const;
 
   friend std::ostream &operator<<(std::ostream &os,