feat(mam): Send mam message from given chid

The current implemetaion of MAM service is a data structure which is
similar to linked-list. This structure causes searching time increase
linearly, so if the user is searching some information with a large
N, then then elapsing time would be really long, too.
Therefore, in this PR, tangle-accelerator starts to support searching
MAM message from a given channel ID which can dramatically reduce
searching time with a proper given channel ID.

Close #610

Author: howjmay

accelerator/core/mam_core.c

@@ -12,6 +12,11 @@
 
 #define MAM_LOGGER "mam_core"
 
+typedef struct channel_info_s {
+  int32_t ch_mss_depth;
+  tryte_t *chid;
+} channel_info_t;
+
 static logger_id_t logger_id;
 
 void ta_mam_logger_init() { logger_id = logger_helper_enable(MAM_LOGGER, LOGGER_DEBUG, true); }
@@ -230,7 +235,7 @@ static status_t ta_mam_init(mam_api_t *const api, const iota_config_t *const ico
 }
 
 static status_t create_channel_fetch_all_transactions(const iota_client_service_t *const service, mam_api_t *const api,
-                                                      const size_t channel_depth, tryte_t *const chid,
+                                                      const size_t channel_depth, bool first_iter, tryte_t *const chid,
                                                       hash81_array_p tag_array) {
   status_t ret = SC_OK;
   find_transactions_req_t *txn_req = find_transactions_req_new();
@@ -241,10 +246,14 @@ static status_t create_channel_fetch_all_transactions(const iota_client_service_
     goto done;
   }
 
-  if (mam_api_channel_create(api, channel_depth, chid) != RC_OK) {
-    ret = SC_MAM_FAILED_CREATE_OR_GET_ID;
-    ta_log_error("%s\n", ta_error_to_string(ret));
-    goto done;
+  // We have created a channel when we want to get to the given channel at the first loop in
+  // `ta_mam_written_msg_to_bundle()`, so we don't need to create a channel once again.
+  if (!first_iter) {
+    if (mam_api_channel_create(api, channel_depth, chid) != RC_OK) {
+      ret = SC_MAM_FAILED_CREATE_OR_GET_ID;
+      ta_log_error("%s\n", "SC_MAM_FAILED_CREATE_OR_GET_ID");
+      goto done;
+    }
   }
 
   flex_trit_t chid_flex_trit[NUM_TRITS_ADDRESS];
@@ -271,6 +280,35 @@ static status_t create_channel_fetch_all_transactions(const iota_client_service_
   return ret;
 }
 
+static bool is_setting_changed(const iota_client_service_t *const service, mam_api_t *const api,
+                               const size_t channel_depth, tryte_t *const chid) {
+  bool changed = true;
+  find_transactions_req_t *txn_req = find_transactions_req_new();
+  transaction_array_t *txn_res = transaction_array_new();
+
+  if (mam_api_channel_create(api, channel_depth, chid) != RC_OK) {
+    ta_log_error("%s\n", ta_error_to_string(SC_MAM_FAILED_CREATE_OR_GET_ID));
+    goto done;
+  }
+
+  flex_trit_t chid_flex_trit[NUM_TRITS_ADDRESS];
+  flex_trits_from_trytes(chid_flex_trit, NUM_TRITS_ADDRESS, chid, NUM_TRYTES_ADDRESS, NUM_TRYTES_ADDRESS);
+  hash243_queue_push(&txn_req->addresses, chid_flex_trit);
+  // TODO use `ta_find_transaction_objects(service, txn_req, txn_res)` instead of the original entangled function
+  retcode_t ret_rc = iota_client_find_transaction_objects(service, txn_req, txn_res);
+  if (ret_rc && ret_rc != RC_NULL_PARAM) {
+    ta_log_error("%s\n", SC_MAM_FAILED_DESTROYED);
+    goto done;
+  }
+
+  changed = (transaction_array_len(txn_res) == 0);
+
+done:
+  find_transactions_req_free(&txn_req);
+  transaction_array_free(txn_res);
+  return changed;
+}
+
 /**
  * @brief Write payload to bundle on the smallest secret key.
  *
@@ -290,27 +328,62 @@ static status_t create_channel_fetch_all_transactions(const iota_client_service_
  * @return return code
  */
 static status_t ta_mam_written_msg_to_bundle(const iota_client_service_t *const service, mam_api_t *const api,
-                                             const size_t channel_depth, mam_encrypt_key_t mam_key,
+                                             const channel_info_t *channel_info, mam_encrypt_key_t mam_key,
                                              char const *const payload, bundle_transactions_t **bundle,
                                              tryte_t *const chid, tryte_t *const msg_id,
                                              mam_send_operation_t *mam_operation) {
   status_t ret = SC_OK;
-  if (!service || !api || !chid || !msg_id || channel_depth < 1) {
+  if (!service || !api || !chid || !msg_id || !channel_info || channel_info->ch_mss_depth < 1) {
     ret = SC_MAM_NULL;
     ta_log_error("%s\n", ta_error_to_string(ret));
     return ret;
   }
   trit_t msg_id_trits[MAM_MSG_ID_SIZE];
   hash81_array_p tag_array = hash81_array_new();
+
+  // Get to the assigned beginning channel ID. If the initial setting is different, then tangle-accelerator won't be
+  // able to generate same chid.
+  if (channel_info->chid) {
+    // The current setting hasn't been used on Tangle, so we should take the normal procedure.
+    if (is_setting_changed(service, api, channel_info->ch_mss_depth, chid)) {
+      goto end_find_starting_chid;
+    }
+
+    int cnt = 0;
+    while (memcmp(channel_info->chid, chid, NUM_TRYTES_ADDRESS)) {
+      if (mam_api_channel_create(api, channel_info->ch_mss_depth, chid) != RC_OK) {
+        ret = SC_MAM_FAILED_CREATE_OR_GET_ID;
+        ta_log_error("%s\n", ta_error_to_string(ret));
+        goto done;
+      }
+
+      if (cnt++ > 100) {
+        ret = SC_MAM_EXCEEDED_CHID_ITER;
+        ta_log_error("%s\n", ta_error_to_string(ret));
+        goto done;
+      }
+    }
+
+  end_find_starting_chid:
+    ta_log_debug("%s\n", "Finish finding starting chid");
+  }
+
+  // If a starting chid is provided, then we don't need to create a new chid in the first iteration of the following
+  // loop.
+  // FIXME: We should figure out a way to avoid passing 'first_iter_with_given_chid' to
+  // 'create_channel_fetch_all_transactions()'
+  bool first_iter_with_given_chid = (bool)channel_info->chid;
   for (;;) {
     hash_array_free(tag_array);
     tag_array = hash81_array_new();
 
-    ret = create_channel_fetch_all_transactions(service, api, channel_depth, chid, tag_array);
+    ret = create_channel_fetch_all_transactions(service, api, channel_info->ch_mss_depth, first_iter_with_given_chid,
+                                                chid, tag_array);
     if (ret) {
       ta_log_error("%s\n", ta_error_to_string(ret));
       goto done;
     }
+    first_iter_with_given_chid = false;
 
     /*
      * Three cases should be considered:
@@ -320,7 +393,7 @@ static status_t ta_mam_written_msg_to_bundle(const iota_client_service_t *const
      */
 
     // Calculate the smallest available msg_ord
-    const int ch_leaf_num = 1 << channel_depth;
+    const int ch_leaf_num = 1 << channel_info->ch_mss_depth;
     int ch_remain_key_num = ch_leaf_num;
     int used_key_num = hash_array_len(tag_array);
     do {
@@ -447,11 +520,6 @@ static status_t ta_mam_api_bundle_read(mam_api_t *const api, bundle_transactions
  * External functions
  ***********************************************************************************************************/
 
-void bundle_transactions_renew(bundle_transactions_t **bundle) {
-  bundle_transactions_free(bundle);
-  bundle_transactions_new(bundle);
-}
-
 status_t ta_send_mam_message(const ta_config_t *const info, const iota_config_t *const iconf,
                              const iota_client_service_t *const service, ta_send_mam_req_t const *const req,
                              ta_send_mam_res_t *const res) {
@@ -473,10 +541,10 @@ status_t ta_send_mam_message(const ta_config_t *const info, const iota_config_t
   mam_send_operation_t mam_operation;
   while (!msg_sent) {
     bundle_transactions_renew(&bundle);
+    struct channel_info_s channel_info = {.ch_mss_depth = data->ch_mss_depth, .chid = data->chid};
 
-    // Create channel merkle tree and find the smallest unused secret key.
     // Write both Header and Packet into one single bundle.
-    ret = ta_mam_written_msg_to_bundle(service, &mam, data->ch_mss_depth, mam_key, data->message, &bundle, chid, msg_id,
+    ret = ta_mam_written_msg_to_bundle(service, &mam, &channel_info, mam_key, data->message, &bundle, chid, msg_id,
                                        &mam_operation);
     if (ret == SC_OK) {
       msg_sent = true;

accelerator/core/mam_core.h

@@ -46,14 +46,6 @@ typedef struct mam_encrypt_key_s {
   mam_ntru_pk_t_set_t ntru_pks;
 } mam_encrypt_key_t;
 
-/**
- * @brief Renew the given bundle
- *
- * @param bundle[in,out] The bundle that will be renewed
- *
- */
-void bundle_transactions_renew(bundle_transactions_t** bundle);
-
 /**
  * @brief Send a MAM message.
  *

accelerator/core/request/ta_send_mam.c

@@ -36,6 +36,7 @@ status_t send_mam_req_v1_init(ta_send_mam_req_t* req) {
 
   send_mam_data_mam_v1_t* data = req->data;
   data->seed = NULL;
+  data->chid = NULL;
   data->message = NULL;
   data->ch_mss_depth = 6;
 
@@ -50,6 +51,7 @@ static void send_mam_req_v1_free(ta_send_mam_req_t** req) {
   if ((*req)->data) {
     send_mam_data_mam_v1_t* data = (*req)->data;
     free(data->seed);
+    free(data->chid);
     free(data->message);
     free((*req)->data);
     (*req)->data = NULL;

accelerator/core/request/ta_send_mam.h

@@ -45,6 +45,8 @@ typedef struct send_mam_data_mam_v1_s {
   tryte_t* seed;
   /** Optional. The depth of channel merkle tree. */
   int32_t ch_mss_depth;
+  /** Optional. The Channel ID which tangle-accelerator starts to search available message slot. */
+  tryte_t* chid;
   /** Required. The message will be append to the channel. */
   char* message;
 } send_mam_data_mam_v1_t;

accelerator/core/serializer/serializer.c

@@ -808,7 +808,7 @@ static status_t send_mam_message_mam_v1_req_deserialize(cJSON const* const json_
   if (json_value != NULL) {
     size_t seed_size = strnlen(json_value->valuestring, NUM_TRYTES_ADDRESS);
 
-    if (seed_size != NUM_TRYTES_HASH) {
+    if (seed_size != NUM_TRYTES_ADDRESS) {
       ret = SC_SERIALIZER_INVALID_REQ;
       ta_log_error("%s\n", ta_error_to_string(ret));
       goto done;
@@ -817,6 +817,21 @@ static status_t send_mam_message_mam_v1_req_deserialize(cJSON const* const json_
     snprintf((char*)data->seed, seed_size + 1, "%s", json_value->valuestring);
   }
 
+  if (cJSON_HasObjectItem(json_key, "chid")) {
+    json_value = cJSON_GetObjectItemCaseSensitive(json_key, "chid");
+    if (json_value != NULL) {
+      size_t chid_size = strnlen(json_value->valuestring, NUM_TRYTES_ADDRESS);
+
+      if (chid_size != NUM_TRYTES_ADDRESS) {
+        ret = SC_SERIALIZER_INVALID_REQ;
+        ta_log_error("%s\n", ta_error_to_string(ret));
+        goto done;
+      }
+      data->chid = (tryte_t*)malloc(sizeof(tryte_t) * (NUM_TRYTES_ADDRESS + 1));
+      snprintf((char*)data->chid, chid_size + 1, "%s", json_value->valuestring);
+    }
+  }
+
   json_value = cJSON_GetObjectItemCaseSensitive(json_key, "message");
   if (json_value != NULL) {
     size_t payload_size = strlen(json_value->valuestring);

common/ta_errors.h

@@ -156,6 +156,8 @@ typedef enum {
   /**< Can't find message in the assign bundle */
   SC_MAM_INVAID_CHID_OR_EPID = 0x0E | SC_MODULE_MAM | SC_SEVERITY_FATAL,
   /**< Failed to add trusted channel ID or endpoint ID */
+  SC_MAM_EXCEEDED_CHID_ITER = 0x0F | SC_MODULE_MAM | SC_SEVERITY_FATAL,
+  /**< Too much iteration for finding a starting chid */
 
   // response module
   SC_RES_NULL = 0x01 | SC_MODULE_RES | SC_SEVERITY_FATAL,

tests/api/mam_test.c

@@ -136,6 +136,50 @@ void test_write_until_next_channel(void) {
   }
 }
 
+void test_write_with_chid(void) {
+  const int beginning_msg_num = 1;
+  char seed[NUM_TRYTES_ADDRESS + 1] = {};
+  const char* json_template_send =
+      "{\"x-api-key\":\"" TEST_TOKEN
+      "\",\"data\":{\"seed\":\"%s\",\"ch_mss_depth\":1,\"message\":\"%s:%d\"}, \"protocol\":\"MAM_V1\"}";
+  const char payload[] = "This is test payload number";
+  const int len = strlen(json_template_send) + NUM_TRYTES_ADDRESS + strlen(payload) + 2;
+  gen_rand_trytes(NUM_TRYTES_ADDRESS, (tryte_t*)seed);
+  double sum = 0;
+  ta_send_mam_res_t* res;
+  test_time_start(&start_time);
+  char* json_result = NULL;
+  for (int i = 0; i < beginning_msg_num; ++i) {
+    res = send_mam_res_new();
+    char* json = (char*)malloc(sizeof(char) * len);
+    snprintf(json, len, json_template_send, seed, payload, i);
+    TEST_ASSERT_EQUAL_INT32(
+        SC_OK, api_send_mam_message(&ta_core.ta_conf, &ta_core.iota_conf, &ta_core.iota_service, json, &json_result));
+    free(json);
+    if (i != beginning_msg_num - 1) {
+      free(json_result);
+      send_mam_res_free(&res);
+    }
+  }
+  send_mam_res_deserialize(json_result, res);
+  free(json_result);
+
+  // Send message from next channel ID
+  const char* json_template_send_chid =
+      "{\"x-api-key\":\"" TEST_TOKEN
+      "\",\"data\":{\"seed\":\"%s\",\"chid\":\"%s\",\"ch_mss_depth\":2,\"message\":\"%s\"}, \"protocol\":\"MAM_V1\"}";
+  const int len_send_chid = strlen(json_template_send_chid) + NUM_TRYTES_ADDRESS * 2 + strlen(payload);
+  char* json_send_chid = (char*)malloc(sizeof(char) * (len_send_chid + 1));
+  snprintf(json_send_chid, len_send_chid, json_template_send_chid, seed, res->chid1, payload);
+  TEST_ASSERT_EQUAL_INT32(SC_OK, api_send_mam_message(&ta_core.ta_conf, &ta_core.iota_conf, &ta_core.iota_service,
+                                                      json_send_chid, &json_result));
+  test_time_end(&start_time, &end_time, &sum);
+  send_mam_res_free(&res);
+  free(json_send_chid);
+  free(json_result);
+  printf("Elapsed time of write_with_chid: %lf\n", sum);
+}
+
 int main(int argc, char* argv[]) {
   UNITY_BEGIN();
   rand_trytes_init();
@@ -158,6 +202,7 @@ int main(int argc, char* argv[]) {
   RUN_TEST(test_send_mam_message);
   RUN_TEST(test_receive_mam_message);
   RUN_TEST(test_write_until_next_channel);
+  RUN_TEST(test_write_with_chid);
   ta_core_destroy(&ta_core);
   return UNITY_END();
 }

tests/unit-test/test_serializer.c

@@ -361,8 +361,8 @@ void test_recv_mam_message_response_deserialize(void) {
 
 void test_send_mam_message_request_deserialize(void) {
   const char* json =
-      "{\"x-api-key\":\"" TEST_TOKEN "\",\"data\":{\"seed\":\"" TRYTES_81_1 "\",\"message\":\"" TEST_PAYLOAD
-      "\",\"ch_mss_depth\":" STR(TEST_CH_DEPTH) ",\"ep_mss_depth\":" STR(
+      "{\"x-api-key\":\"" TEST_TOKEN "\",\"data\":{\"seed\":\"" TRYTES_81_1 "\",\"chid\":\"" TEST_ADDRESS
+      "\",\"message\":\"" TEST_PAYLOAD "\",\"ch_mss_depth\":" STR(TEST_CH_DEPTH) ",\"ep_mss_depth\":" STR(
           TEST_EP_DEPTH) "},\"key\":{\"ntru\":[\"" TEST_NTRU_PK "\"],\"psk\":[\"" TRYTES_81_2 "\",\"" TRYTES_81_3
                          "\"]}, \"protocol\":\"MAM_V1\"}";
 
@@ -371,7 +371,8 @@ void test_send_mam_message_request_deserialize(void) {
   send_mam_data_mam_v1_t* data = (send_mam_data_mam_v1_t*)req->data;
 
   TEST_ASSERT_EQUAL_STRING(TEST_TOKEN, req->service_token);
-  TEST_ASSERT_EQUAL_MEMORY(TRYTES_81_1, data->seed, NUM_TRYTES_HASH);
+  TEST_ASSERT_EQUAL_STRING(TRYTES_81_1, data->seed);
+  TEST_ASSERT_EQUAL_STRING(TEST_ADDRESS, data->chid);
   TEST_ASSERT_EQUAL_INT(TEST_CH_DEPTH, data->ch_mss_depth);
   TEST_ASSERT_EQUAL_STRING(TEST_PAYLOAD, data->message);
   TEST_ASSERT_EQUAL_STRING(TEST_NTRU_PK, mamv1_ntru_key_at(req, 0));

utils/bundle_array.h

@@ -23,6 +23,17 @@ extern "C" {
  * `bundle_transactions_t` objects. It provides an easier way to save and traverse all the bundles.
  */
 
+/**
+ * @brief Renew the given bundle
+ *
+ * @param bundle[in,out] The bundle that will be renewed
+ *
+ */
+static inline void bundle_transactions_renew(bundle_transactions_t **bundle) {
+  bundle_transactions_free(bundle);
+  bundle_transactions_new(bundle);
+}
+
 typedef UT_array bundle_array_t;
 // We should synchronize this implementation as to the implementation in the IOTA library
 static UT_icd bundle_transactions_icd = {sizeof(iota_transaction_t), 0, 0, 0};