feat: add bytes support for asset modules (#12548)

* feat: add bytes support for asset modules

* docs

Author: LingyuCoder

crates/rspack_core/src/lib.rs

@@ -196,6 +196,7 @@ pub enum ModuleType {
   AssetInline,
   AssetResource,
   AssetSource,
+  AssetBytes,
   Asset,
   Runtime,
   Remote,
@@ -261,6 +262,7 @@ impl ModuleType {
       ModuleType::AssetSource => "asset/source",
       ModuleType::AssetResource => "asset/resource",
       ModuleType::AssetInline => "asset/inline",
+      ModuleType::AssetBytes => "asset/bytes",
       ModuleType::Runtime => "runtime",
       ModuleType::Remote => "remote-module",
       ModuleType::Fallback => "fallback-module",
@@ -301,6 +303,7 @@ impl From<&str> for ModuleType {
       "asset/resource" => Self::AssetResource,
       "asset/source" => Self::AssetSource,
       "asset/inline" => Self::AssetInline,
+      "asset/bytes" => Self::AssetBytes,
 
       custom => Self::Custom(custom.into()),
     }

crates/rspack_core/src/runtime_globals.rs

@@ -260,10 +260,12 @@ bitflags! {
     const AMD_DEFINE = 1 << 67;
     const AMD_OPTIONS = 1 << 68;
 
+    const TO_BINARY = 1 << 69;
+
     // defer import support
-    const ASYNC_MODULE_EXPORT_SYMBOL = 1 << 69;
-    const MAKE_DEFERRED_NAMESPACE_OBJECT = 1 << 70;
-    const MAKE_DEFERRED_NAMESPACE_OBJECT_SYMBOL = 1 << 71;
+    const ASYNC_MODULE_EXPORT_SYMBOL = 1 << 70;
+    const MAKE_DEFERRED_NAMESPACE_OBJECT = 1 << 71;
+    const MAKE_DEFERRED_NAMESPACE_OBJECT_SYMBOL = 1 << 72;
   }
 }
 
@@ -355,6 +357,7 @@ pub fn runtime_globals_to_string(
     RuntimeGlobals::HAS_CSS_MODULES => "has css modules".to_string(),
 
     RuntimeGlobals::HAS_FETCH_PRIORITY => "has fetch priority".to_string(),
+    RuntimeGlobals::TO_BINARY => format!("{scope_name}.tb"),
     _ => unreachable!(),
   }
 }

crates/rspack_plugin_asset/src/lib.rs

@@ -48,6 +48,7 @@ const DEFAULT_ENCODING: &str = "base64";
 enum DataUrlOptions {
   Inline(bool),
   Source,
+  Bytes,
   Auto(Option<AssetParserDataUrl>),
 }
 
@@ -60,6 +61,7 @@ const ASSET_RESOURCE: bool = false;
 #[derive(Debug, Clone)]
 pub enum CanonicalizedDataUrlOption {
   Source,
+  Bytes,
   Asset(IsInline),
 }
 
@@ -68,6 +70,10 @@ impl CanonicalizedDataUrlOption {
     matches!(self, CanonicalizedDataUrlOption::Source)
   }
 
+  pub fn is_bytes(&self) -> bool {
+    matches!(self, CanonicalizedDataUrlOption::Bytes)
+  }
+
   pub fn is_inline(&self) -> bool {
     matches!(self, CanonicalizedDataUrlOption::Asset(ASSET_INLINE))
   }
@@ -118,6 +124,14 @@ impl AssetParserAndGenerator {
     }
   }
 
+  pub fn with_bytes() -> Self {
+    Self {
+      emit: false,
+      data_url: DataUrlOptions::Bytes,
+      parsed_asset_config: None,
+    }
+  }
+
   fn decode_data_uri_content(encoding: &str, content: &str, source: &BoxSource) -> Vec<u8> {
     if encoding == "base64"
       && let Some(cleaned) = base64::clean_base64(content)
@@ -372,7 +386,7 @@ impl ParserAndGenerator for AssetParserAndGenerator {
     if self
       .parsed_asset_config
       .as_ref()
-      .is_some_and(|x| x.is_source() || x.is_inline())
+      .is_some_and(|x| x.is_source() || x.is_inline() || x.is_bytes())
       || !self.emit
     {
       if source_types.is_empty() {
@@ -420,7 +434,9 @@ impl ParserAndGenerator for AssetParserAndGenerator {
 
         let parsed_size = self.parsed_asset_config.as_ref().map(|config| {
           match config {
-            CanonicalizedDataUrlOption::Source => original_source_size,
+            CanonicalizedDataUrlOption::Source | CanonicalizedDataUrlOption::Bytes => {
+              original_source_size
+            }
             CanonicalizedDataUrlOption::Asset(meta) => {
               match *meta {
                 ASSET_INLINE => {
@@ -465,6 +481,7 @@ impl ParserAndGenerator for AssetParserAndGenerator {
 
     self.parsed_asset_config = match &self.data_url {
       DataUrlOptions::Source => Some(CanonicalizedDataUrlOption::Source),
+      DataUrlOptions::Bytes => Some(CanonicalizedDataUrlOption::Bytes),
       DataUrlOptions::Inline(val) => Some(CanonicalizedDataUrlOption::Asset(*val)),
       DataUrlOptions::Auto(option) => {
         let limit_size = parse_context
@@ -524,7 +541,30 @@ impl ParserAndGenerator for AssetParserAndGenerator {
 
     match generate_context.requested_source_type {
       SourceType::JavaScript | SourceType::CssUrl => {
-        let exported_content = if parsed_asset_config.is_inline() {
+        let exported_content = if parsed_asset_config.is_bytes() {
+          let mut encoded_source = base64::encode_to_string(source.buffer());
+          if generate_context.requested_source_type == SourceType::CssUrl {
+            encoded_source = format!("data:application/octet-stream;base64,{}", encoded_source);
+            generate_context
+              .data
+              .insert(CodeGenerationDataUrl::new(encoded_source.clone()));
+            serde_json::to_string(&encoded_source).to_rspack_result()?
+          } else {
+            generate_context
+              .runtime_requirements
+              .insert(RuntimeGlobals::REQUIRE_SCOPE);
+            generate_context
+              .runtime_requirements
+              .insert(RuntimeGlobals::TO_BINARY);
+            format!(
+              "{}({})",
+              compilation
+                .runtime_template
+                .render_runtime_globals(&RuntimeGlobals::TO_BINARY),
+              serde_json::to_string(&encoded_source).to_rspack_result()?
+            )
+          }
+        } else if parsed_asset_config.is_inline() {
           let resource_data: &ResourceData = normal_module.resource_resolved_data();
           let data_url = module_generator_options.and_then(|x| x.asset_data_url());
           let encoded_source: String;
@@ -899,6 +939,11 @@ impl Plugin for AssetPlugin {
       Box::new(move |_, _| Box::new(AssetParserAndGenerator::with_source())),
     );
 
+    ctx.register_parser_and_generator_builder(
+      rspack_core::ModuleType::AssetBytes,
+      Box::new(move |_, _| Box::new(AssetParserAndGenerator::with_bytes())),
+    );
+
     Ok(())
   }
 }

crates/rspack_plugin_runtime/src/runtime_module/mod.rs

@@ -42,6 +42,7 @@ mod runtime_id;
 mod startup_chunk_dependencies;
 mod startup_entrypoint;
 mod system_context;
+mod to_binary;
 mod utils;
 pub use amd_define::AmdDefineRuntimeModule;
 pub use amd_options::AmdOptionsRuntimeModule;
@@ -87,4 +88,5 @@ pub use runtime_id::RuntimeIdRuntimeModule;
 pub use startup_chunk_dependencies::StartupChunkDependenciesRuntimeModule;
 pub use startup_entrypoint::StartupEntrypointRuntimeModule;
 pub use system_context::SystemContextRuntimeModule;
+pub use to_binary::ToBinaryRuntimeModule;
 pub use utils::*;

crates/rspack_plugin_runtime/src/runtime_module/runtime/to_binary.ejs

@@ -0,0 +1,23 @@
+// define to binary helper
+<%- TO_BINARY %> = <% if (_is_neutral_platform) { %>typeof Buffer !== 'undefined' ? <% } %>
+  <% if (_is_node_platform || _is_neutral_platform) { %>
+    <%- returningFunction("new Uint8Array(Buffer.from(base64, 'base64'))", "base64") %>
+  <% } %>
+  <% if (_is_neutral_platform) { %> : <% } %>
+  <% if (_is_web_platform || _is_neutral_platform) { %>
+(<%- basicFunction("") %> {
+	var table = new Uint8Array(128);
+	for (var i = 0; i < 64; i++) table[i < 26 ? i + 65 : i < 52 ? i + 71 : i < 62 ? i - 4 : i * 4 - 205] = i;
+	return <%- basicFunction("base64") %> {
+		var n = base64.length, bytes = new Uint8Array((n - (base64[n - 1] == '=') - (base64[n - 2] == '=')) * 3 / 4 | 0);
+		for (var i = 0, j = 0; i < n;) {
+			var c0 = table[base64.charCodeAt(i++)], c1 = table[base64.charCodeAt(i++)];
+			var c2 = table[base64.charCodeAt(i++)], c3 = table[base64.charCodeAt(i++)];
+			bytes[j++] = (c0 << 2) | (c1 >> 4);
+			bytes[j++] = (c1 << 4) | (c2 >> 2);
+			bytes[j++] = (c2 << 6) | c3;
+		}
+		return bytes;
+	};
+})()
+<% } %>

crates/rspack_plugin_runtime/src/runtime_module/to_binary.rs

@@ -0,0 +1,48 @@
+use rspack_collections::Identifier;
+use rspack_core::{Compilation, RuntimeModule, RuntimeTemplate, impl_runtime_module};
+
+#[impl_runtime_module]
+#[derive(Debug)]
+pub struct ToBinaryRuntimeModule {
+  id: Identifier,
+}
+
+impl ToBinaryRuntimeModule {
+  pub fn new(runtime_template: &RuntimeTemplate) -> Self {
+    Self::with_default(Identifier::from(format!(
+      "{}to_binary",
+      runtime_template.runtime_module_prefix()
+    )))
+  }
+}
+
+#[async_trait::async_trait]
+impl RuntimeModule for ToBinaryRuntimeModule {
+  fn name(&self) -> Identifier {
+    self.id
+  }
+
+  fn template(&self) -> Vec<(String, String)> {
+    vec![(
+      self.id.to_string(),
+      include_str!("runtime/to_binary.ejs").to_string(),
+    )]
+  }
+
+  async fn generate(&self, compilation: &Compilation) -> rspack_error::Result<String> {
+    let is_node_platform = compilation.platform.is_node();
+    let is_web_platform = compilation.platform.is_web();
+    let is_neutral_platform = compilation.platform.is_neutral();
+
+    let source = compilation.runtime_template.render(
+      &self.id,
+      Some(serde_json::json!({
+        "_is_node_platform": is_node_platform,
+        "_is_web_platform": is_web_platform,
+        "_is_neutral_platform": is_neutral_platform,
+      })),
+    )?;
+
+    Ok(source)
+  }
+}

crates/rspack_plugin_runtime/src/runtime_plugin.rs

@@ -35,7 +35,8 @@ use crate::{
     LoadScriptRuntimeModule, MakeDeferredNamespaceObjectRuntimeModule,
     MakeNamespaceObjectRuntimeModule, NodeModuleDecoratorRuntimeModule, NonceRuntimeModule,
     OnChunkLoadedRuntimeModule, PublicPathRuntimeModule, RelativeUrlRuntimeModule,
-    RuntimeIdRuntimeModule, SystemContextRuntimeModule, chunk_has_css, is_enabled_for_chunk,
+    RuntimeIdRuntimeModule, SystemContextRuntimeModule, ToBinaryRuntimeModule, chunk_has_css,
+    is_enabled_for_chunk,
   },
 };
 
@@ -601,6 +602,12 @@ async fn runtime_requirements_in_tree(
             .boxed(),
         )?;
       }
+      RuntimeGlobals::TO_BINARY => {
+        compilation.add_runtime_module(
+          chunk_ukey,
+          ToBinaryRuntimeModule::new(&compilation.runtime_template).boxed(),
+        )?;
+      }
       _ => {}
     }
   }

packages/rspack-test-tools/src/runner/node/index.ts

@@ -191,6 +191,7 @@ export class NodeRunner implements ITestRunner {
 			},
 			WAITING: this.globalContext!.WAITING,
 			process,
+			TextDecoder,
 			URL,
 			Blob,
 			Symbol,

packages/rspack/src/config/adapter.ts

@@ -695,6 +695,7 @@ function getRawGeneratorOptions(
 	if (
 		[
 			"asset/source",
+			"asset/bytes",
 			"javascript",
 			"javascript/auto",
 			"javascript/dynamic",

packages/rspack/src/config/defaults.ts

@@ -583,6 +583,10 @@ const applyModuleDefaults = (
 			{
 				with: { type: "text" },
 				type: "asset/source"
+			},
+			{
+				with: { type: "bytes" },
+				type: "asset/bytes"
 			}
 		);