Merge remote-tracking branch 'origin/master' into fcitx

Author: Fcitx Bot

src/converter/BUILD.bazel

@@ -57,12 +57,15 @@ mozc_cc_library(
         "//rewriter:__pkg__",
     ],
     deps = [
+        ":attribute",
+        ":inner_segment",
         ":lattice",
         "//base:number_util",
         "//base:util",
         "//base:vlog",
         "//base/container:freelist",
         "//base/strings:assign",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/log",
         "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/strings",
@@ -90,13 +93,59 @@ mozc_cc_test(
     size = "small",
     srcs = ["candidate_test.cc"],
     deps = [
+        ":inner_segment",
         ":segments",
         "//base:number_util",
         "//testing:gunit_main",
         "@com_google_absl//absl/strings",
     ],
 )
 
+mozc_cc_library(
+    name = "attribute",
+    hdrs = ["attribute.h"],
+    visibility = [
+        "//engine:__pkg__",
+        "//prediction:__pkg__",
+        "//rewriter:__pkg__",
+    ],
+)
+
+mozc_cc_library(
+    name = "inner_segment",
+    hdrs = [
+        "inner_segment.h",
+    ],
+    visibility = [
+        "//engine:__pkg__",
+        "//prediction:__pkg__",
+        "//request:__pkg__",
+        "//rewriter:__pkg__",
+    ],
+    deps = [
+        "@com_google_absl//absl/container:fixed_array",
+        "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+mozc_cc_test(
+    name = "inner_segment_test",
+    srcs = [
+        "inner_segment_test.cc",
+    ],
+    deps = [
+        ":inner_segment",
+        "//testing:gunit_main",
+        "@com_google_absl//absl/log",
+        "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
 mozc_cc_library(
     name = "segments_matchers",
     testonly = 1,
@@ -212,6 +261,7 @@ mozc_cc_library(
     deps = [
         ":candidate_filter",
         ":connector",
+        ":inner_segment",
         ":lattice",
         ":node",
         ":segmenter",
@@ -412,6 +462,7 @@ mozc_cc_test(
     ],
     deps = [
         ":immutable_converter_no_factory",
+        ":inner_segment",
         ":lattice",
         ":node",
         ":segments",
@@ -464,6 +515,7 @@ mozc_cc_library(
         ":converter_interface",
         ":history_reconstructor",
         ":immutable_converter_interface",
+        ":inner_segment",
         ":reverse_converter",
         ":segments",
         "//base:util",
@@ -498,6 +550,7 @@ mozc_cc_test(
         ":converter_interface",
         ":immutable_converter_interface",
         ":immutable_converter_no_factory",
+        ":inner_segment",
         ":segments",
         ":segments_matchers",
         "//base:util",

src/converter/attribute.h

@@ -0,0 +1,105 @@
+// Copyright 2010-2021, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef MOZC_CONVERTER_ATTRIBUTE_H_
+#define MOZC_CONVERTER_ATTRIBUTE_H_
+
+namespace mozc {
+namespace converter {
+
+// This unscoped enum is used to allow implicit conversion to integer types for
+// bitwise operations. This is an alternative to using an enum class and
+// explicit casting.
+class Attribute {
+ public:
+  // TODO(taku): Can use unnamed enum after full migration.
+  enum Attribute_ {
+    DEFAULT_ATTRIBUTE = 0,
+    // this was the best candidate before learning
+    BEST_CANDIDATE = 1 << 0,
+    // this candidate was reranked by user
+    RERANKED = 1 << 1,
+    // don't save it in history
+    NO_HISTORY_LEARNING = 1 << 2,
+    // don't save it in suggestion
+    NO_SUGGEST_LEARNING = 1 << 3,
+    // NO_HISTORY_LEARNING | NO_SUGGEST_LEARNING
+    NO_LEARNING = (1 << 2 | 1 << 3),
+    // learn it with left/right context
+    CONTEXT_SENSITIVE = 1 << 4,
+    // has "did you mean"
+    SPELLING_CORRECTION = 1 << 5,
+    // No need to have full/half width expansion
+    NO_VARIANTS_EXPANSION = 1 << 6,
+    // No need to have extra descriptions
+    NO_EXTRA_DESCRIPTION = 1 << 7,
+    // was generated by real-time conversion
+    REALTIME_CONVERSION = 1 << 8,
+    // contains tokens in user dictionary.
+    USER_DICTIONARY = 1 << 9,
+    // command candidate. e.g., incognito mode.
+    COMMAND_CANDIDATE = 1 << 10,
+    // key characters are consumed partially.
+    // Consumed size is |consumed_key_size|.
+    // If not set, all the key characters are consumed.
+    PARTIALLY_KEY_CONSUMED = 1 << 11,
+    // Typing correction candidate.
+    // - Special description should be shown when the candidate is created
+    //   by a dictionary predictor.
+    // - No description should be shown when the candidate is loaded from
+    //   history.
+    // - Otherwise following unexpected behavior can be observed.
+    //   1. Type "やんしょん" and submit "マンション" (annotated with "補正").
+    //   2. Type "まんしょん".
+    //   3. "マンション" (annotated with "補正") is shown as a candidate
+    //      regardless of a user's correct typing.
+    TYPING_CORRECTION = 1 << 12,
+    // Auto partial suggestion candidate.
+    // - Special description should be shown when the candidate is created
+    //   by a dictionary predictor.
+    // - No description should be shown when the candidate is loaded from
+    //   history.
+    AUTO_PARTIAL_SUGGESTION = 1 << 13,
+    // Predicted from user prediction history.
+    USER_HISTORY_PREDICTION = 1 << 14,
+    // Contains suffix dictionary.
+    SUFFIX_DICTIONARY = 1 << 15,
+    // Disables modification and removal in rewriters.
+    NO_MODIFICATION = 1 << 16,
+    // Candidate which is reranked by user segment history rewriter.
+    USER_SEGMENT_HISTORY_REWRITER = 1 << 17,
+    // Keys are expanded in the dictionary lookup. Usually
+    // Kana-modifiers are expanded.
+    KEY_EXPANDED_IN_DICTIONARY = 1 << 18,
+  };
+};
+}  // namespace converter
+}  // namespace mozc
+
+#endif  // MOZC_CONVERTER_ATTRIBUTE_H_

src/converter/candidate.cc

@@ -44,6 +44,7 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/span.h"
 #include "base/number_util.h"
+#include "converter/inner_segment.h"
 
 #ifdef MOZC_CANDIDATE_DEBUG
 #include "absl/strings/str_cat.h"
@@ -85,54 +86,6 @@ void Candidate::Dlog(absl::string_view filename, int line,
 }
 #endif  // MOZC_CANDIDATE_DEBUG
 
-bool Candidate::IsValid() const {
-  if (inner_segment_boundary.empty()) {
-    return true;
-  }
-  // The sums of the lengths of key, value components must coincide with those
-  // of key, value, respectively.
-  size_t sum_key_len = 0, sum_value_len = 0;
-  for (InnerSegmentIterator iter(this); !iter.Done(); iter.Next()) {
-    sum_key_len += iter.GetKey().size();
-    sum_value_len += iter.GetValue().size();
-  }
-  return sum_key_len == key.size() && sum_value_len == value.size();
-}
-
-bool Candidate::EncodeLengths(size_t key_len, size_t value_len,
-                              size_t content_key_len, size_t content_value_len,
-                              uint32_t *result) {
-  if (key_len > std::numeric_limits<uint8_t>::max() ||
-      value_len > std::numeric_limits<uint8_t>::max() ||
-      content_key_len > std::numeric_limits<uint8_t>::max() ||
-      content_value_len > std::numeric_limits<uint8_t>::max()) {
-    return false;
-  }
-  *result = (static_cast<uint32_t>(key_len) << 24) |
-            (static_cast<uint32_t>(value_len) << 16) |
-            (static_cast<uint32_t>(content_key_len) << 8) |
-            static_cast<uint32_t>(content_value_len);
-  return true;
-}
-
-std::tuple<size_t, size_t, size_t, size_t> Candidate::DecodeLengths(
-    uint32_t encoded) {
-  return std::make_tuple(encoded >> 24, (encoded >> 16) & 0xff,
-                         (encoded >> 8) & 0xff, (encoded & 0xff));
-}
-
-bool Candidate::PushBackInnerSegmentBoundary(size_t key_len, size_t value_len,
-                                             size_t content_key_len,
-                                             size_t content_value_len) {
-  uint32_t encoded;
-  if (EncodeLengths(key_len, value_len, content_key_len, content_value_len,
-                    &encoded)) {
-    inner_segment_boundary.push_back(encoded);
-    return true;
-  }
-  return false;
-}
-
 std::string Candidate::DebugString() const {
   std::stringstream os;
   os << "(key=" << key << " ckey=" << content_key << " val=" << value
@@ -151,70 +104,15 @@ std::string Candidate::DebugString() const {
   }
   if (!inner_segment_boundary.empty()) {
     os << " segbdd=";
-    for (size_t i = 0; i < inner_segment_boundary.size(); ++i) {
-      const uint32_t encoded_lengths = inner_segment_boundary[i];
-      const auto [key_len, value_len, content_key_len, content_value_len] =
-          DecodeLengths(encoded_lengths);
-      os << absl::StreamFormat("<%d,%d,%d,%d>", key_len, value_len,
-                               content_key_len, content_value_len);
+    for (const auto &iter : inner_segments()) {
+      os << absl::StreamFormat(
+          "<%d,%d,%d,%d>", iter.GetKey().size(), iter.GetValue().size(),
+          iter.GetContentKey().size(), iter.GetContentValue().size());
     }
   }
   os << ")" << std::endl;
   return os.str();
 }
 
-void Candidate::InnerSegmentIterator::Next() {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_++];
-  key_offset_ += encoded_lengths >> 24;
-  value_offset_ += (encoded_lengths >> 16) & 0xff;
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetKey() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(key_offset_, encoded_lengths >> 24);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetValue() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(value_offset_, (encoded_lengths >> 16) & 0xff);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetContentKey() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(key_offset_, (encoded_lengths >> 8) & 0xff);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetContentValue() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  return absl::string_view(value_offset_, encoded_lengths & 0xff);
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetFunctionalKey() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  const int key_len = encoded_lengths >> 24;
-  const int content_key_len = (encoded_lengths >> 8) & 0xff;
-  if (const int key_size = key_len - content_key_len; key_size > 0) {
-    return absl::string_view(key_offset_ + content_key_len, key_size);
-  }
-  return absl::string_view();
-}
-
-absl::string_view Candidate::InnerSegmentIterator::GetFunctionalValue() const {
-  DCHECK_LT(index_, inner_segment_boundary_.size());
-  const uint32_t encoded_lengths = inner_segment_boundary_[index_];
-  const int value_len = (encoded_lengths >> 16) & 0xff;
-  const int content_value_len = encoded_lengths & 0xff;
-  if (const int value_size = value_len - content_value_len; value_size > 0) {
-    return absl::string_view(value_offset_ + content_value_len, value_size);
-  }
-  return absl::string_view();
-}
-
 }  // namespace converter
 }  // namespace mozc