Merge remote-tracking branch 'origin/master' into fcitx

Author: Fcitx Bot

src/MODULE.bazel

@@ -82,6 +82,14 @@ bazel_dep(
     name = "rules_pkg",
     version = "1.1.0",
 )
+single_version_override(
+    module_name = "rules_pkg",
+    patches = [
+        # Patch to fix the failure of Bazel fetch.
+        "bazel/rules_pkg_MODULE.bazel.patch",
+    ],
+    version = "1.1.0",
+)
 
 # rules_python: 1.4.1 2025-05-09
 # https://github.com/bazelbuild/rules_python/

src/bazel/rules_pkg_MODULE.bazel.patch

@@ -0,0 +1,10 @@
+--- MODULE.bazel
++++ MODULE.bazel
+@@ -36,7 +36,3 @@ register_toolchains(
+
+ local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository")
+
+-local_repository(
+-    name = "mappings_test_external_repo",
+-    path = "tests/mappings/external_repo",
+-)

src/converter/converter.cc

@@ -305,15 +305,15 @@ void Converter::FinishConversion(const ConversionRequest &request,
       absl::IntervalClosed, bitgen, 1, std::numeric_limits<uint64_t>::max());
   segments->set_revert_id(revert_id);
 
+  const prediction::Result history_result = MakeHistoryResult(*segments);
+  const std::vector<prediction::Result> committed_results =
+      MakeLearningResults(*segments);
   const ConversionRequest finish_req = ConversionRequestBuilder()
                                            .SetConversionRequestView(request)
-                                           .SetHistorySegmentsView(*segments)
+                                           .SetHistoryResultView(history_result)
                                            .Build();
-  DCHECK(finish_req.HasConverterHistorySegments());
-
   rewriter_->Finish(finish_req, *segments);
-  predictor_->Finish(finish_req, MakeLearningResults(*segments),
-                     segments->revert_id());
+  predictor_->Finish(finish_req, committed_results, segments->revert_id());
 
   if (request.request_type() != ConversionRequest::CONVERSION &&
       segments->conversion_segments_size() >= 1 &&
@@ -711,17 +711,21 @@ bool Converter::PredictForRequestWithSegments(const ConversionRequest &request,
   DCHECK(segments);
   DCHECK(predictor_);
 
+  const prediction::Result history_result = MakeHistoryResult(*segments);
+
   const ConversionRequest conv_req = ConversionRequestBuilder()
                                          .SetConversionRequestView(request)
-                                         .SetHistorySegmentsView(*segments)
+                                         .SetHistoryResultView(history_result)
                                          .Build();
-  DCHECK(conv_req.HasConverterHistorySegments());
 
   const std::vector<prediction::Result> results = predictor_->Predict(conv_req);
 
   Segment *segment = segments->mutable_conversion_segment(0);
   DCHECK(segment);
 
+  // TODO(taku): Make utility functions to convert
+  // Segments <-> history_result, committed_results.
+
   for (const prediction::Result &result : results) {
     Candidate *candidate = segment->add_candidate();
     strings::Assign(candidate->key, result.key);
@@ -842,5 +846,42 @@ std::vector<prediction::Result> Converter::MakeLearningResults(
 
   return results;
 }
+
+// static
+prediction::Result Converter::MakeHistoryResult(const Segments &segments) {
+  prediction::Result result;
+
+  if (segments.history_segments_size() == 0) {
+    return result;
+  }
+
+  bool inner_segment_boundary_failed = false;
+  for (const auto &segment : segments.history_segments()) {
+    if (segment.candidates_size() == 0) {
+      return prediction::Result::DefaultResult();  // Returns an empty result.
+    }
+    const Candidate &candidate = segment.candidate(0);
+    absl::StrAppend(&result.key, candidate.key);
+    absl::StrAppend(&result.value, candidate.value);
+    result.candidate_attributes |= candidate.attributes;
+    uint32_t encoded = 0;
+    if (!Candidate::EncodeLengths(candidate.key.size(), candidate.value.size(),
+                                  candidate.content_key.size(),
+                                  candidate.content_value.size(), &encoded)) {
+      inner_segment_boundary_failed = true;
+    }
+    result.inner_segment_boundary.emplace_back(encoded);
+  }
+
+  if (inner_segment_boundary_failed) result.inner_segment_boundary.clear();
+
+  const int size = segments.history_segments_size();
+  result.lid = segments.history_segment(0).candidate(0).lid;
+  result.rid = segments.history_segment(size - 1).candidate(0).rid;
+  result.cost = segments.history_segment(size - 1).candidate(0).cost;
+
+  return result;
+}
+
 }  // namespace converter
 }  // namespace mozc

src/converter/converter.h

@@ -173,10 +173,13 @@ class Converter final : public ConverterInterface {
     return *modules_;
   }
 
-  // Utility method to results for Predictor::Finish().
+  // Utility method to make committed results for Predictor::Finish().
   static std::vector<prediction::Result> MakeLearningResults(
       const Segments &segments);
 
+  // Utility method to make history result passed to ConversionRequest.
+  static prediction::Result MakeHistoryResult(const Segments &segments);
+
  private:
   Converter() = default;
 

src/converter/converter_test.cc

@@ -856,13 +856,14 @@ TEST_F(ConverterTest, StartSuggestion) {
     ConversionRequest::Options options = {.request_type =
                                               ConversionRequest::SUGGESTION};
     Segments segments;
-    const ConversionRequest request = ConversionRequestBuilder()
-                                          .SetComposer(composer)
-                                          .SetRequestView(client_request)
-                                          .SetConfigView(config)
-                                          .SetHistorySegmentsView(segments)
-                                          .SetOptions(std::move(options))
-                                          .Build();
+    const ConversionRequest request =
+        ConversionRequestBuilder()
+            .SetComposer(composer)
+            .SetRequestView(client_request)
+            .SetConfigView(config)
+            .SetHistoryResult(Converter::MakeHistoryResult(segments))
+            .SetOptions(std::move(options))
+            .Build();
 
     EXPECT_TRUE(converter->StartPrediction(request, &segments));
     EXPECT_EQ(segments.segments_size(), 1);
@@ -881,13 +882,14 @@ TEST_F(ConverterTest, StartSuggestion) {
     ConversionRequest::Options options = {.request_type =
                                               ConversionRequest::SUGGESTION};
     Segments segments;
-    const ConversionRequest request = ConversionRequestBuilder()
-                                          .SetComposer(composer)
-                                          .SetRequestView(client_request)
-                                          .SetConfigView(config)
-                                          .SetHistorySegmentsView(segments)
-                                          .SetOptions(std::move(options))
-                                          .Build();
+    const ConversionRequest request =
+        ConversionRequestBuilder()
+            .SetComposer(composer)
+            .SetRequestView(client_request)
+            .SetConfigView(config)
+            .SetHistoryResult(Converter::MakeHistoryResult(segments))
+            .SetOptions(std::move(options))
+            .Build();
 
     EXPECT_TRUE(converter->StartPrediction(request, &segments));
     EXPECT_EQ(segments.segments_size(), 1);
@@ -1027,7 +1029,7 @@ TEST_F(ConverterTest, PredictSetKey) {
     const ConversionRequest request =
         ConversionRequestBuilder()
             .SetComposer(composer)
-            .SetHistorySegmentsView(segments)
+            .SetHistoryResult(Converter::MakeHistoryResult(segments))
             .SetRequestType(ConversionRequest::PREDICTION)
             .Build();
     ASSERT_TRUE(converter->StartPrediction(request, &segments));
@@ -1132,7 +1134,7 @@ TEST_F(ConverterTest, ComposerKeySelection) {
     const ConversionRequest request =
         ConversionRequestBuilder()
             .SetComposer(composer)
-            .SetHistorySegmentsView(segments)
+            .SetHistoryResult(Converter::MakeHistoryResult(segments))
             .SetOptions(
                 {.composer_key_selection = ConversionRequest::CONVERSION_KEY})
             .Build();
@@ -1149,7 +1151,7 @@ TEST_F(ConverterTest, ComposerKeySelection) {
     const ConversionRequest request =
         ConversionRequestBuilder()
             .SetComposer(composer)
-            .SetHistorySegmentsView(segments)
+            .SetHistoryResult(Converter::MakeHistoryResult(segments))
             .SetOptions(
                 {.composer_key_selection = ConversionRequest::PREDICTION_KEY})
             .Build();
@@ -1183,11 +1185,12 @@ TEST_F(ConverterTest, SuppressionDictionaryForRewriter) {
   composer.InsertCharacter("placeholder");
   commands::Context context;
   Segments segments;
-  const ConversionRequest request = ConversionRequestBuilder()
-                                        .SetComposer(composer)
-                                        .SetHistorySegmentsView(segments)
-                                        .SetConfig(config)
-                                        .Build();
+  const ConversionRequest request =
+      ConversionRequestBuilder()
+          .SetComposer(composer)
+          .SetHistoryResult(Converter::MakeHistoryResult(segments))
+          .SetConfig(config)
+          .Build();
   EXPECT_TRUE(converter->StartConversion(request, &segments));
 
   // Verify that words inserted by the rewriter is suppressed if its in the
@@ -1386,11 +1389,12 @@ TEST_F(ConverterTest, LimitCandidatesSize) {
   mozc::composer::Composer composer(table, request_proto, config);
   composer.InsertCharacterPreedit("あ");
   Segments segments;
-  const ConversionRequest request1 = ConversionRequestBuilder()
-                                         .SetComposer(composer)
-                                         .SetHistorySegmentsView(segments)
-                                         .SetRequest(request_proto)
-                                         .Build();
+  const ConversionRequest request1 =
+      ConversionRequestBuilder()
+          .SetComposer(composer)
+          .SetHistoryResult(Converter::MakeHistoryResult(segments))
+          .SetRequest(request_proto)
+          .Build();
   ASSERT_TRUE(converter->StartConversion(request1, &segments));
   ASSERT_EQ(segments.conversion_segments_size(), 1);
   const int original_candidates_size = segments.segment(0).candidates_size();
@@ -1402,11 +1406,12 @@ TEST_F(ConverterTest, LimitCandidatesSize) {
 
   segments.Clear();
   request_proto.set_candidates_size_limit(original_candidates_size - 1);
-  const ConversionRequest request2 = ConversionRequestBuilder()
-                                         .SetComposer(composer)
-                                         .SetRequest(request_proto)
-                                         .SetHistorySegmentsView(segments)
-                                         .Build();
+  const ConversionRequest request2 =
+      ConversionRequestBuilder()
+          .SetComposer(composer)
+          .SetRequest(request_proto)
+          .SetHistoryResult(Converter::MakeHistoryResult(segments))
+          .Build();
   ASSERT_TRUE(converter->StartConversion(request2, &segments));
   ASSERT_EQ(segments.conversion_segments_size(), 1);
   EXPECT_GE(original_candidates_size - 1,
@@ -1418,11 +1423,12 @@ TEST_F(ConverterTest, LimitCandidatesSize) {
 
   segments.Clear();
   request_proto.set_candidates_size_limit(0);
-  const ConversionRequest request3 = ConversionRequestBuilder()
-                                         .SetComposer(composer)
-                                         .SetRequest(request_proto)
-                                         .SetHistorySegmentsView(segments)
-                                         .Build();
+  const ConversionRequest request3 =
+      ConversionRequestBuilder()
+          .SetComposer(composer)
+          .SetRequest(request_proto)
+          .SetHistoryResult(Converter::MakeHistoryResult(segments))
+          .Build();
   ASSERT_TRUE(converter->StartConversion(request3, &segments));
   ASSERT_EQ(segments.conversion_segments_size(), 1);
   EXPECT_EQ(segments.segment(0).candidates_size(), 1);
@@ -1477,7 +1483,7 @@ TEST_F(ConverterTest, UserEntryInMobilePrediction) {
             .SetComposer(composer)
             .SetRequestView(request)
             .SetConfigView(config)
-            .SetHistorySegmentsView(segments)
+            .SetHistoryResult(Converter::MakeHistoryResult(segments))
             .SetOptions(std::move(options))
             .Build();
     EXPECT_TRUE(converter->StartPrediction(conversion_request, &segments));
@@ -1668,7 +1674,7 @@ TEST_F(ConverterTest, RewriterShouldRespectDefaultCandidates) {
           .SetComposer(composer)
           .SetRequestView(request)
           .SetConfigView(config)
-          .SetHistorySegmentsView(segments)
+          .SetHistoryResult(Converter::MakeHistoryResult(segments))
           .SetOptions(std::move(options))
           .Build();
 
@@ -1782,7 +1788,7 @@ TEST_F(ConverterTest, DoNotAddOverlappingNodesForPrediction) {
           .SetComposer(composer)
           .SetRequestView(request)
           .SetConfigView(config)
-          .SetHistorySegmentsView(segments)
+          .SetHistoryResult(Converter::MakeHistoryResult(segments))
           .SetOptions(std::move(options))
           .Build();
 
@@ -2437,6 +2443,42 @@ TEST_F(ConverterTest, MakeLearningResultsTest) {
   }
 }
 
+TEST_F(ConverterTest, MakeHistoryResultTest) {
+  Segments segments;
+  for (int i = 0; i < 3; ++i) {
+    Segment *segment = segments.add_segment();
+    segment->set_segment_type(Segment::HISTORY);
+    Candidate *c = segment->add_candidate();
+    c->key = absl::StrCat("k", i);
+    c->content_key = "k";
+    c->value = absl::StrCat("v", i);
+    c->content_value = "v";
+    c->lid = i;
+    c->rid = i + 1;
+    c->cost = i;
+  };
+
+  const prediction::Result result = Converter::MakeHistoryResult(segments);
+  EXPECT_EQ(result.key, "k0k1k2");
+  EXPECT_EQ(result.value, "v0v1v2");
+  EXPECT_EQ(result.lid, segments.segment(0).candidate(0).lid);
+  EXPECT_EQ(result.rid, segments.segment(2).candidate(0).rid);
+  EXPECT_EQ(result.cost, 2);  // only the last cost.
+
+  int n = 0;
+  for (Candidate::InnerSegmentIterator iter(result.inner_segment_boundary,
+                                            result.key, result.value);
+       !iter.Done(); iter.Next()) {
+    const Candidate &c = segments.segment(n).candidate(0);
+    EXPECT_EQ(iter.GetKey(), c.key);
+    EXPECT_EQ(iter.GetContentKey(), c.content_key);
+    EXPECT_EQ(iter.GetValue(), c.value);
+    EXPECT_EQ(iter.GetContentValue(), c.content_value);
+    ++n;
+  }
+  EXPECT_EQ(n, 3);
+}
+
 TEST_F(ConverterTest, Bugfix424676259) {
   auto mock_predictor = absl::make_unique<MockPredictor>();
   auto mock_rewriter = absl::make_unique<MockRewriter>();

src/prediction/BUILD.bazel

@@ -206,13 +206,15 @@ mozc_cc_library(
     visibility = [
         "//converter:__subpackages__",
         "//engine:__subpackages__",
+        "//request:__subpackages__",
     ],
     deps = [
         ":zero_query_dict",
         "//base/strings:unicode",
         "//composer:query",
         "//converter:segments",
         "//dictionary:dictionary_token",
+        "@com_google_absl//absl/base:no_destructor",
         "@com_google_absl//absl/base:nullability",
         "@com_google_absl//absl/log",
         "@com_google_absl//absl/strings",
@@ -623,6 +625,7 @@ mozc_cc_library(
     deps = [
         ":result",
         "//base:util",
+        "//base/strings:assign",
         "//converter:converter_interface",
         "//converter:immutable_converter_interface",
         "//converter:segments",

src/prediction/dictionary_prediction_aggregator.cc

@@ -134,7 +134,7 @@ bool IsLongKeyForRealtimeCandidates(const ConversionRequest &request) {
 //  "一二", "十二", "壱拾弐", etc for 12.
 // TODO(toshiyuki): Define the spec and support Kanji.
 std::optional<std::string> GetNumberHistory(const ConversionRequest &request) {
-  const std::string history_value = request.converter_history_value(1);
+  absl::string_view history_value = request.converter_history_value(1);
   if (history_value.empty() || !NumberUtil::IsArabicNumber(history_value)) {
     return std::nullopt;
   }
@@ -752,8 +752,8 @@ void DictionaryPredictionAggregator::AggregateZeroQuery(
 
   // There are 4 sources in zero query suggestion.
 
-  const std::string history_value = request.converter_history_value(1);
-  const std::string history_key = request.converter_history_key(1);
+  absl::string_view history_value = request.converter_history_value(1);
+  absl::string_view history_key = request.converter_history_key(1);
 
   if (history_key.empty() || history_value.empty()) {
     return;
@@ -908,8 +908,8 @@ void DictionaryPredictionAggregator::AggregateBigram(
 
   // TODO(toshiyuki): Support suggestion from the last 2 histories.
   //  ex) "六本木"+"ヒルズ"->"レジデンス"
-  const std::string history_key = request.converter_history_key(1);
-  const std::string history_value = request.converter_history_value(1);
+  absl::string_view history_key = request.converter_history_key(1);
+  absl::string_view history_value = request.converter_history_value(1);
   if (history_key.empty() || history_value.empty()) {
     return;
   }