ds plugins OPTIMIZE pass req xpaths to load_cb

load_cb can be xpath aware for operational datastore, and fetch only the
required data for the datastore plugins that support it.

Implemented support:
1. redis DS plugin supports this for all datastores.
2. mongo ds plugin supports this except for operational datastore.
  - ietf-origin metadata must be explicitly retrieved for each path.
  - this is not implemented yet, so all data will be retrieved.
3. json ds plugin - unimplemented. ignores passed xpaths.

This has the benefit of optimally loading only the requested data, and
not constructing the entire module datatree, if only a small part is
being read.

Further, it can reduce the amount of time modules remain read locked,
when they have large amount of data, and occasional readers of small
data.

Author: irfanHaslanded

src/modinfo.c

@@ -1376,7 +1376,7 @@ sr_module_oper_data_load(struct sr_mod_info_mod_s *mod, sr_conn_ctx_t *conn, sr_
             }
             /* load push oper data for the session from the datastore plugin if cache was empty */
             if (!mod_data && (err_info = sr_module_file_data_append(mod->ly_mod, mod->ds_handle, SR_DS_OPERATIONAL,
-                    oper_push_dup[i].cid, oper_push_dup[i].sid, NULL, 0, &mod_data))) {
+                    oper_push_dup[i].cid, oper_push_dup[i].sid, mod->xpaths, mod->xpath_count, &mod_data))) {
                 goto cleanup;
             }
         } else {

src/plugins/ds_mongo.c

@@ -396,13 +396,14 @@ srpds_get_access(mongoc_collection_t *module, char **owner, char **group, mode_t
  * @param[in] ctx Libyang context.
  * @param[in] xpaths Array of XPaths.
  * @param[in] xpath_cnt XPath count.
+ * @param[in] oper_ds Flag if the filter is for loading operational data and special handling is needed.
  * @param[out] is_valid Whether the @p xpath_filter is valid.
  * @param[out] xpath_filter XPath filter for the database.
  * @return NULL on success;
  * @return Sysrepo error info on error.
  */
 static sr_error_info_t *
-srpds_process_load_paths(struct ly_ctx *ctx, const char **xpaths, uint32_t xpath_cnt, int *is_valid, bson_t *xpath_filter)
+srpds_process_load_paths(struct ly_ctx *ctx, const char **xpaths, uint32_t xpath_cnt, int oper_ds, int *is_valid, bson_t *xpath_filter)
 {
     sr_error_info_t *err_info = NULL;
     uint32_t i;
@@ -416,6 +417,17 @@ srpds_process_load_paths(struct ly_ctx *ctx, const char **xpaths, uint32_t xpath
     bson_init(xpath_filter);
     bson_append_array_begin(xpath_filter, "$or", 3, &top);
 
+    if (oper_ds) {
+        /*
+         * TODO
+         * xpath filtering is unimplemented for MONGO DS operational datastore.
+         * /sysrepo:discard-items and metadata (ietf-origin) needs to be explicitly retrieved.
+         *
+         * So, just fetch all the data until then.
+         */
+        goto cleanup;
+    }
+
     /* create new data node for lyd_find_path to work correctly */
     if (lyd_new_path(NULL, ctx, "/ietf-yang-library:yang-library", NULL, 0, &ctx_node) != LY_SUCCESS) {
         ERRINFO(&err_info, plugin_name, SR_ERR_LY, "lyd_new_path()", "");
@@ -3399,7 +3411,7 @@ srpds_mongo_load(const struct lys_module *mod, sr_datastore_t ds, sr_cid_t cid,
         goto cleanup;
     }
 
-    if ((err_info = srpds_process_load_paths(mod->ctx, xpaths, xpath_count, &is_valid, &xpath_filter))) {
+    if ((err_info = srpds_process_load_paths(mod->ctx, xpaths, xpath_count, (ds == SR_DS_OPERATIONAL), &is_valid, &xpath_filter))) {
         goto cleanup;
     }
 

src/plugins/ds_redis.c

@@ -2641,12 +2641,13 @@ srpds_load_and_copy_data(redisContext *ctx, const char *mod_ns_src, const char *
  * @param[in] ctx Libyang context.
  * @param[in] xpaths Array of XPaths.
  * @param[in] xpath_cnt XPath count.
+ * @param[in] oper_ds Flag if the filter is for loading operational data and special handling is needed.
  * @param[out] xpath_filter Final query filter.
  * @return NULL on success;
  * @return Sysrepo error info on error.
  */
 static sr_error_info_t *
-srpds_process_load_paths(struct ly_ctx *ctx, const char **xpaths, uint32_t xpath_cnt, char **xpath_filter)
+srpds_process_load_paths(struct ly_ctx *ctx, const char **xpaths, uint32_t xpath_cnt, int oper_ds, char **xpath_filter)
 {
     sr_error_info_t *err_info = NULL;
     uint32_t i;
@@ -2740,6 +2741,19 @@ srpds_process_load_paths(struct ly_ctx *ctx, const char **xpaths, uint32_t xpath
         path = NULL;
     }
 
+    if (xpath_cnt && oper_ds) {
+        /* explicitly add discard-items to the query for operational datastore */
+        if ((err_info = srpds_escape_string(plugin_name, "/sysrepo:discard-items", &escaped_path))) {
+            goto cleanup;
+        }
+        if (asprintf(&tmp, "%s | %s*", *xpath_filter, escaped_path) == -1) {
+            ERRINFO(&err_info, plugin_name, SR_ERR_NO_MEMORY, "asprintf()", strerror(errno));
+            goto cleanup;
+        }
+        free(*xpath_filter);
+        *xpath_filter = tmp;
+    }
+
 cleanup:
     if (err_info) {
         free(*xpath_filter);
@@ -3897,7 +3911,7 @@ srpds_redis_load(const struct lys_module *mod, sr_datastore_t ds, sr_cid_t cid,
         goto cleanup;
     }
 
-    if ((err_info = srpds_process_load_paths(mod->ctx, xpaths, xpath_count, &xpath_filter))) {
+    if ((err_info = srpds_process_load_paths(mod->ctx, xpaths, xpath_count, (ds == SR_DS_OPERATIONAL), &xpath_filter))) {
         goto cleanup;
     }