perf(build): improve git submodules installation speed (#50)

Author: linrongbin16

README.md

@@ -70,8 +70,6 @@ And multiple trigger timings (colorschemes don't have end time):
 1. Don't lazy this plugin, it only takes ~4 ms to load.
 2. Load this plugin before all other start plugins.
 
-**Warning:** hold your wifi, it `clone` and `pull` a lot of git repos!
-
 ### [lazy.nvim](https://github.com/folke/lazy.nvim)
 
 ```lua
@@ -98,6 +96,15 @@ require('pckr').add({
 })
 ```
 
+If you have issues on running multiple git clone/pull commands, try set `concurrency=1` in the `update` API:
+
+```lua
+require('colorbox').update({
+    --- @type integer
+    concurrency = 4,
+})
+```
+
 ## 🔧 Configuration
 
 ```lua
@@ -128,9 +135,9 @@ require('colorbox').setup({
     --- @type "dark"|"light"|nil
     background = nil,
 
-    -- cache dir
-    -- for macos/linux: $HOME/.local/share/nvim/colorbox.nvim
-    -- for windows: $env:USERPROFILE\AppData\Local\nvim-data\colorbox.nvim
+    -- Cache directory
+    -- * For macos/linux: $HOME/.local/share/nvim/colorbox.nvim
+    -- * For windows: $env:USERPROFILE\AppData\Local\nvim-data\colorbox.nvim
     --
     --- @type string
     cache_dir = string.format("%s/colorbox.nvim", vim.fn.stdpath('data')),

lua/colorbox.lua

@@ -191,7 +191,7 @@ end
 
 local function _policy()
     if Configs.background == "dark" or Configs.background == "light" then
-        vim.cmd(string.format([[set background=%s]], Configs.background))
+        vim.opt.background = Configs.background
     end
     if Configs.policy == "shuffle" then
         _policy_shuffle()
@@ -263,7 +263,13 @@ local function setup(opts)
     _timing()
 end
 
-local function update()
+--- @param opts {concurrency:integer}?
+local function update(opts)
+    opts = opts or { concurrency = 4 }
+    opts.concurrency = type(opts.concurrency) == "number"
+            and math.max(opts.concurrency, 1)
+        or 4
+
     logger.setup({
         name = "colorbox",
         level = LogLevels.DEBUG,
@@ -282,22 +288,42 @@ local function update()
     )
     vim.opt.packpath:append(home_dir)
 
-    local jobs = {}
     local HandleToColorSpecsMap =
         require("colorbox.db").get_handle_to_color_specs_map()
+
+    -- a list of job params
+    --- @type colorbox.Options[]
+    local jobs_pending_queue = {}
+
+    -- a list of job id
+    --- @type integer[]
+    local jobs_working_queue = {}
+
+    -- job id to job params map
+    --- @type table<integer, colorbox.Options>
+    local jobid_to_jobs_map = {}
+
+    local prepared_count = 0
+    local finished_count = 0
+
+    for handle, spec in pairs(HandleToColorSpecsMap) do
+        prepared_count = prepared_count + 1
+    end
+    logger.info("started %s jobs", vim.inspect(prepared_count))
+
     for handle, spec in pairs(HandleToColorSpecsMap) do
         local function _on_output(chanid, data, name)
             if type(data) == "table" then
                 logger.debug(
-                    "%s (%s): %s",
-                    vim.inspect(handle),
+                    "(%s) %s: %s",
                     vim.inspect(name),
+                    vim.inspect(handle),
                     vim.inspect(data)
                 )
                 local lines = {}
-                for _, line in ipairs(data) do
-                    if string.len(vim.trim(line)) > 0 then
-                        table.insert(lines, line)
+                for _, d in ipairs(data) do
+                    if type(d) == "string" and string.len(vim.trim(d)) > 0 then
+                        table.insert(lines, d)
                     end
                 end
                 if #lines > 0 then
@@ -307,47 +333,103 @@ local function update()
         end
         local function _on_exit(jid, exitcode, name)
             logger.debug(
-                "%s (%s-%s): exit with %s",
-                vim.inspect(handle),
-                vim.inspect(jid),
+                "(%s-%s) %s: exit with %s",
                 vim.inspect(name),
+                vim.inspect(jid),
+                vim.inspect(handle),
                 vim.inspect(exitcode)
             )
+
+            local removed_from_working_queue = false
+            for i, working_jobid in ipairs(jobs_working_queue) do
+                if working_jobid == jid then
+                    table.remove(jobs_working_queue, i)
+                    removed_from_working_queue = true
+                    break
+                end
+            end
+            if not removed_from_working_queue then
+                logger.err(
+                    "failed to remove job id %s from jobs_working_queue: %s",
+                    vim.inspect(jid),
+                    vim.inspect(jobs_working_queue)
+                )
+            end
+            if jobid_to_jobs_map[jid] == nil then
+                logger.err(
+                    "failed to remove job id %s from jobid_to_jobs_map: %s",
+                    vim.inspect(jid),
+                    vim.inspect(jobid_to_jobs_map)
+                )
+            end
+            jobid_to_jobs_map[jid] = nil
+
+            if #jobs_pending_queue > 0 then
+                local waiting_job_param = jobs_pending_queue[1]
+                table.remove(jobs_pending_queue, 1)
+
+                local new_jobid = vim.fn.jobstart(
+                    waiting_job_param.cmd,
+                    waiting_job_param.opts
+                )
+                table.insert(jobs_working_queue, new_jobid)
+                jobid_to_jobs_map[new_jobid] = waiting_job_param
+
+                finished_count = finished_count + 1
+            else
+                logger.info("finished %s jobs", vim.inspect(finished_count))
+                logger.close_file_mode_w()
+            end
         end
 
+        local param = nil
         if
             vim.fn.isdirectory(spec.full_pack_path) > 0
             and vim.fn.isdirectory(spec.full_pack_path .. "/.git") > 0
         then
-            local cmd = { "git", "pull" }
-            logger.debug("update command:%s", vim.inspect(cmd))
-            local jobid = vim.fn.jobstart(cmd, {
-                cwd = spec.full_pack_path,
-                stdout_buffered = true,
-                stderr_buffered = true,
-                on_stdout = _on_output,
-                on_stderr = _on_output,
-                on_exit = _on_exit,
-            })
-            table.insert(jobs, jobid)
+            param = {
+                handle = handle,
+                cmd = { "git", "pull" },
+                opts = {
+                    cwd = spec.full_pack_path,
+                    detach = true,
+                    stdout_buffered = true,
+                    stderr_buffered = true,
+                    on_stdout = _on_output,
+                    on_stderr = _on_output,
+                    on_exit = _on_exit,
+                },
+            }
         else
-            local cmd =
-                { "git", "clone", "--depth=1", spec.url, spec.pack_path }
-            logger.debug("install command:%s", vim.inspect(cmd))
-            local jobid = vim.fn.jobstart(cmd, {
-                cwd = home_dir,
-                stdout_buffered = true,
-                stderr_buffered = true,
-                on_stdout = _on_output,
-                on_stderr = _on_output,
-                on_exit = _on_exit,
-            })
-            logger.debug("installing %s", vim.inspect(handle))
-            table.insert(jobs, jobid)
+            param = {
+                handle = handle,
+                cmd = { "git", "clone", "--depth=1", spec.url, spec.pack_path },
+                opts = {
+                    cwd = home_dir,
+                    detach = true,
+                    stdout_buffered = true,
+                    stderr_buffered = true,
+                    on_stdout = _on_output,
+                    on_stderr = _on_output,
+                    on_exit = _on_exit,
+                },
+            }
+        end
+
+        table.insert(jobs_pending_queue, param)
+
+        if #jobs_working_queue < opts.concurrency then
+            local waiting_job_param = jobs_pending_queue[1]
+            table.remove(jobs_pending_queue, 1)
+
+            local new_jobid =
+                vim.fn.jobstart(waiting_job_param.cmd, waiting_job_param.opts)
+            table.insert(jobs_working_queue, new_jobid)
+            jobid_to_jobs_map[new_jobid] = waiting_job_param
+
+            finished_count = finished_count + 1
         end
     end
-    vim.fn.jobwait(jobs)
-    logger.close_file_mode_w()
 end
 
 local M = { setup = setup, update = update }