[USING, DON'T MERGE THO] Enable Lexical-style parsing

Cargo.toml

@@ -12,8 +12,15 @@ crate-type = ["cdylib"]
 
 [dependencies]
 lib0 = "0.16.10"
-yrs = "0.16.10"
+yrs = { git = "https://github.com/its-dart/y-crdt.git", branch = "v0.16.10" }
+
+[dev-dependencies]
+assert-json-diff = "2.0.2"
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.79", features = ["unbounded_depth"] }
 
 [dependencies.pyo3]
 version = "0.19.2"
-features = ["extension-module"]
+# https://stackoverflow.com/a/71757467
+extension-module = ["pyo3/extension-module"]
+default = ["extension-module"]

src/y_doc.rs

@@ -1,4 +1,5 @@
 use std::cell::RefCell;
+use std::collections::HashMap;
 use std::rc::Rc;
 use std::rc::Weak;
 
@@ -12,9 +13,11 @@ use crate::y_xml::YXmlFragment;
 use crate::y_xml::YXmlText;
 use pyo3::prelude::*;
 use pyo3::types::PyBytes;
+use pyo3::types::PyDict;
 use pyo3::types::PyTuple;
+use yrs::updates::decoder::Decode;
 use yrs::updates::encoder::Encode;
-use yrs::Doc;
+use yrs::*;
 use yrs::OffsetKind;
 use yrs::Options;
 use yrs::SubscriptionId;
@@ -415,6 +418,35 @@ pub fn apply_update(doc: &mut YDoc, diff: Vec<u8>) -> PyResult<()> {
     Ok(())
 }
 
+#[pyfunction]
+pub fn update_to_nodes(diff: Vec<u8>) -> PyObject {
+    let doc = Doc::new();
+    let text = doc.get_or_insert_text("name");
+    let mut txn = doc.transact_mut(); 
+    text.push(&mut txn, "Hello from yrs!");
+    text.format(&mut txn, 11, 3, HashMap::from([
+    ("link".into(), "https://github.com/y-crdt/y-crdt".into())
+    ]));
+
+    let remote_doc = Doc::new();
+
+    // only difference from the basic example--added `mut` and the observer
+    let mut remote_text = remote_doc.get_or_insert_text("name");
+    remote_text.observe_deep(|_txn, _events| {
+        println!("observer triggered");
+    });
+
+    let mut remote_txn = remote_doc.transact_mut();
+    let state_vector = remote_txn.state_vector();
+    let bytes = txn.encode_diff_v1(&state_vector);
+    let update = Update::decode_v1(&bytes).unwrap();
+    remote_txn.apply_update(update);
+    println!("{}", remote_text.get_string(&remote_txn));
+
+    Python::with_gil(|py| PyDict::new(py).into())
+
+}
+
 #[pyclass(unsendable)]
 pub struct AfterTransactionEvent {
     before_state: PyObject,

tests/playground.ts

@@ -0,0 +1,41 @@
+import * as Y from "yjs";
+
+const processNodeRecursive = (node: Y.XmlFragment | Y.Map<Y.Item> | Y.Text): Record<string, any> => {
+  const result = Object.fromEntries([...node._map].map(([k, v]) => [k, v.content.getContent()[0]]));
+  const children: Record<string, any>[]  = [];
+
+  if ("firstChild" in node) {
+    let currentChild = node.firstChild;
+    while (currentChild) {
+      children.push(processNodeRecursive(currentChild));
+      currentChild = currentChild.nextSibling;
+    }
+    result.children = children;
+  }
+
+  if ("toDelta" in node) {
+    node.toDelta().forEach(({ insert }: { insert?: Y.Map<Y.Item> | Y.Text}) => {
+      if (!insert) {
+        return;
+      }
+
+      if (typeof insert === "string") {
+        children[children.length - 1].text = insert;
+        return;
+      }
+
+      children.push(processNodeRecursive(insert));
+    });
+    result.children = children;
+  }
+
+  return result;
+};
+
+const rawStateAsUpdate = '\x01a\x9CµäÏ\x0E\x00(\x01\x04root\x05__dir\x01w\x03ltr\x07\x01\x04root\x06(\x00\x9CµäÏ\x0E\x01\x06__type\x01w\tparagraph(\x00\x9CµäÏ\x0E\x01\b__format\x01}\x00(\x00\x9CµäÏ\x0E\x01\b__indent\x01}\x00(\x00\x9CµäÏ\x0E\x01\x05__dir\x01w\x03ltr\x07\x00\x9CµäÏ\x0E\x01\x01(\x00\x9CµäÏ\x0E\x06\x06__type\x01w\x04text(\x00\x9CµäÏ\x0E\x06\b__format\x01}\x00(\x00\x9CµäÏ\x0E\x06\x07__style\x01w\x00(\x00\x9CµäÏ\x0E\x06\x06__mode\x01}\x00(\x00\x9CµäÏ\x0E\x06\b__detail\x01}\x00\x84\x9CµäÏ\x0E\x06\x01a\x87\x9CµäÏ\x0E\x01\x06(\x00\x9CµäÏ\x0E\r\x06__type\x01w\x04list(\x00\x9CµäÏ\x0E\r\b__format\x01}\x00(\x00\x9CµäÏ\x0E\r\b__indent\x01}\x00!\x00\x9CµäÏ\x0E\r\x05__dir\x01(\x00\x9CµäÏ\x0E\r\n__listType\x01w\x06number(\x00\x9CµäÏ\x0E\r\x05__tag\x01w\x02ol(\x00\x9CµäÏ\x0E\r\x07__start\x01}\x01\x07\x00\x9CµäÏ\x0E\r\x06(\x00\x9CµäÏ\x0E\x15\x06__type\x01w\blistitem(\x00\x9CµäÏ\x0E\x15\b__format\x01}\x00(\x00\x9CµäÏ\x0E\x15\b__indent\x01}\x00!\x00\x9CµäÏ\x0E\x15\x05__dir\x01(\x00\x9CµäÏ\x0E\x15\x07__value\x01}\x01\x01\x00\x9CµäÏ\x0E\x15\x01\x00\x05\x81\x9CµäÏ\x0E\x1B\x01\x84\x9CµäÏ\x0E\f\x01 \x87\x9CµäÏ\x0E"\x01(\x00\x9CµäÏ\x0E#\x06__type\x01w\x04text(\x00\x9CµäÏ\x0E#\b__format\x01}\x01(\x00\x9CµäÏ\x0E#\x07__style\x01w\x00(\x00\x9CµäÏ\x0E#\x06__mode\x01}\x00(\x00\x9CµäÏ\x0E#\b__detail\x01}\x00\x84\x9CµäÏ\x0E#\x01b¡\x9CµäÏ\x0E\x11\x01¡\x9CµäÏ\x0E\x19\x01¨\x9CµäÏ\x0E*\x01w\x03ltr¨\x9CµäÏ\x0E+\x01w\x03ltr\x87\x9CµäÏ\x0E!\x01(\x00\x9CµäÏ\x0E.\x06__type\x01w\x04text(\x00\x9CµäÏ\x0E.\b__format\x01}\x00(\x00\x9CµäÏ\x0E.\x07__style\x01w\x00(\x00\x9CµäÏ\x0E.\x06__mode\x01}\x00(\x00\x9CµäÏ\x0E.\b__detail\x01}\x00\x84\x9CµäÏ\x0E.\x01c\x81\x9CµäÏ\x0E\x15\x01\x00\x05\x87\x9CµäÏ\x0E5\x06(\x00\x9CµäÏ\x0E;\x06__type\x01w\blistitem(\x00\x9CµäÏ\x0E;\b__format\x01}\x00(\x00\x9CµäÏ\x0E;\b__indent\x01}\x00!\x00\x9CµäÏ\x0E;\x05__dir\x01(\x00\x9CµäÏ\x0E;\x07__value\x01}\x02\x07\x00\x9CµäÏ\x0E;\x06(\x00\x9CµäÏ\x0EA\x06__type\x01w\x04list(\x00\x9CµäÏ\x0EA\b__format\x01}\x00(\x00\x9CµäÏ\x0EA\b__indent\x01}\x00(\x00\x9CµäÏ\x0EA\x05__dir\x01w\x03ltr(\x00\x9CµäÏ\x0EA\n__listType\x01w\x06number(\x00\x9CµäÏ\x0EA\x05__tag\x01w\x02ol(\x00\x9CµäÏ\x0EA\x07__start\x01}\x01\x07\x00\x9CµäÏ\x0EA\x06(\x00\x9CµäÏ\x0EI\x06__type\x01w\blistitem(\x00\x9CµäÏ\x0EI\b__format\x01}\x00(\x00\x9CµäÏ\x0EI\b__indent\x01}\x00!\x00\x9CµäÏ\x0EI\x05__dir\x01(\x00\x9CµäÏ\x0EI\x07__value\x01}\x01¨\x9CµäÏ\x0EM\x01w\x03ltr\x07\x00\x9CµäÏ\x0EI\x01(\x00\x9CµäÏ\x0EP\x06__type\x01w\x04text(\x00\x9CµäÏ\x0EP\b__format\x01}\x00(\x00\x9CµäÏ\x0EP\x07__style\x01w\x00(\x00\x9CµäÏ\x0EP\x06__mode\x01}\x00(\x00\x9CµäÏ\x0EP\b__detail\x01}\x00\x84\x9CµäÏ\x0EP\x01d\x81\x9CµäÏ\x0EI\x01\x00\x05\x81\x9CµäÏ\x0E;\x01\x00\x05¨\x9CµäÏ\x0E?\x01~\x87\x9CµäÏ\x0E\r\x06(\x00\x9CµäÏ\x0Ed\x06__type\x01w\tparagraph(\x00\x9CµäÏ\x0Ed\b__format\x01}\x00(\x00\x9CµäÏ\x0Ed\b__indent\x01}\x00!\x00\x9CµäÏ\x0Ed\x05__dir\x01¨\x9CµäÏ\x0Eh\x01w\x03ltr\x07\x00\x9CµäÏ\x0Ed\x01(\x00\x9CµäÏ\x0Ej\x06__type\x01w\x04text(\x00\x9CµäÏ\x0Ej\b__format\x01}\x00(\x00\x9CµäÏ\x0Ej\x07__style\x01w\x00(\x00\x9CµäÏ\x0Ej\x06__mode\x01}\x00(\x00\x9CµäÏ\x0Ej\b__detail\x01}\x00\x84\x9CµäÏ\x0Ej\x01e\x01\x9CµäÏ\x0E\t\x11\x01\x19\x01\x1B\x07*\x025\x06?\x01M\x01W\fh\x01';
+const update = new Uint8Array([...rawStateAsUpdate].map((e) => e.charCodeAt(0)));
+const doc = new Y.Doc();
+Y.applyUpdate(doc, update)
+const root = doc.getXmlFragment("root");
+const result = processNodeRecursive(root);
+console.log(JSON.stringify(result, null, 2));