Merge pull request #105 from drbergman/import-export-intracellular

support import/export of intracellulars

Author: drbergman

src/VCTComponents.jl

@@ -46,7 +46,7 @@ function assembleIntracellular!(cell_to_components_dict::Dict{String,<:Union{Phy
     return assembleIntracellular!(cell_to_vec_components_dict; kwargs...)
 end
 
-function assembleIntracellular!(cell_to_components_dict::Dict{String,Vector{PhysiCellComponent}}; name::String="assembled")
+function assembleIntracellular!(cell_to_components_dict::Dict{String,Vector{PhysiCellComponent}}; name::String="assembled", skip_db_insert::Bool=false)
     #! get all components to assign IDs
     unique_components = PhysiCellComponent[]
     for components in values(cell_to_components_dict)
@@ -145,7 +145,7 @@ function assembleIntracellular!(cell_to_components_dict::Dict{String,Vector{Phys
     end
 
     #! make sure the database is updated, variations.db intialized
-    if initialized
+    if initialized && !skip_db_insert
         insertFolder(:intracellular, splitpath(folder)[end])
     end
 

src/VCTConfiguration.jl

@@ -22,7 +22,7 @@ end
 function retrieveElement(xml_doc::XMLDocument, xml_path::Vector{<:AbstractString}; required::Bool=true)
     current_element = root(xml_doc)
     for path_element in xml_path
-        if !occursin(":",path_element)
+        if !occursin(":", path_element)
             current_element = find_element(current_element, path_element)
             if isnothing(current_element)
                 required ? retrieveElementError(xml_path, path_element) : return nothing

src/VCTExport.jl

@@ -71,8 +71,7 @@ function prepareFolder(simulation::Simulation, export_folder::AbstractString)
 
     #! intracellulars
     if row.intracellular_id[1] != -1
-        path_to_intracellular = joinpath(locationPath(:intracellular, simulation), "intracellular.xml")
-        cp(path_to_intracellular, joinpath(export_folder, "config", "intracellular.xml"))
+        exportIntracellular(simulation, export_folder)
     end
 
     #! ic cells
@@ -125,6 +124,52 @@ function prepareFolder(simulation::Simulation, export_folder::AbstractString)
     return revertSimulationFolderToCurrentPhysiCell(export_folder, physicell_version), physicell_version
 end
 
+function exportIntracellular(simulation::Simulation, export_folder::AbstractString)
+    path_to_intracellular = joinpath(locationPath(:intracellular, simulation), "intracellular.xml")
+    xml_doc = openXML(path_to_intracellular)
+    intracellulars_element = retrieveElement(xml_doc, ["intracellulars"])
+    intracellular_mapping = Dict{String,Tuple{String,String}}()
+    for intracellular_element in child_elements(intracellulars_element)
+        intracellular_id = attribute(intracellular_element, "ID")
+        intracellular_type = attribute(intracellular_element, "type")
+        new_root = child_elements(intracellular_element) |> first
+        new_xml_doc = XMLDocument()
+        set_root(new_xml_doc, new_root)
+        path_end = joinpath("config", "intracellular_$(intracellular_type)_$(intracellular_id).xml")
+        new_path = joinpath(export_folder, path_end)
+        save_file(new_xml_doc, new_path)
+        closeXML(new_xml_doc)
+        intracellular_mapping[intracellular_id] = (intracellular_type, path_end)
+    end
+    
+    config_xml = openXML(joinpath(export_folder, "config", "PhysiCell_settings.xml"))
+
+    cell_definitions_element = retrieveElement(xml_doc, ["cell_definitions"])
+    for cell_definition_element in child_elements(cell_definitions_element)
+        if name(cell_definition_element) != "cell_definition"
+            continue
+        end
+        cell_type = attribute(cell_definition_element, "name")
+        intracellular_ids_element = find_element(cell_definition_element, "intracellular_ids")
+        ID_elements = get_elements_by_tagname(intracellular_ids_element, "ID")
+        @assert length(ID_elements) <= 1 "Do not (yet?) support multiple intracellular models for a single cell type."
+        intracellular_id = ID_elements |> first |> content
+        config_cell_def_intracellular_element = retrieveElement(config_xml, ["cell_definitions", "cell_definition:name:$(cell_type)", "phenotype", "intracellular"])
+        set_attribute(config_cell_def_intracellular_element, "type", intracellular_mapping[intracellular_id][1])
+
+        #! get (or create) the sbml_filename element
+        sbml_filename_element = find_element(config_cell_def_intracellular_element, "sbml_filename")
+        if isnothing(sbml_filename_element)
+            sbml_filename_element = new_child(config_cell_def_intracellular_element, "sbml_filename")
+        end
+        set_content(sbml_filename_element, intracellular_mapping[intracellular_id][2])
+    end
+    
+    closeXML(config_xml)
+    closeXML(xml_doc)
+    return
+end
+
 function revertSimulationFolderToCurrentPhysiCell(export_folder::AbstractString, physicell_version::AbstractString)
     success = revertMain(export_folder, physicell_version)
     success &= revertMakefile(export_folder, physicell_version)
@@ -137,7 +182,9 @@ function revertMain(export_folder::AbstractString, physicell_version::AbstractSt
     path_to_main = joinpath(export_folder, "main.cpp")
     lines = readlines(path_to_main)
     idx = findfirst(x -> contains(x, "<getopt.h>"), lines)
-    popat!(lines, idx)
+    if !isnothing(idx)
+        popat!(lines, idx)
+    end
 
     idx1 = findfirst(x -> contains(x, "// read arguments"), lines)
     if isnothing(idx1)
@@ -245,8 +292,9 @@ function revertConfig(export_folder::AbstractString, physicell_version::Abstract
     set_content(filename_element, "cell_rules.csv")
 
     #! intracellulars
-    #! lol, not supported for export yet
+    #! handled in exportIntracellular
 
+    save_file(xml_doc, path_to_config)
     closeXML(xml_doc)
     return true
 end

src/VCTImport.jl

@@ -33,7 +33,7 @@ struct ImportSources
     ic_dc::ImportSource
 end
 
-function ImportSources(src::Dict)
+function ImportSources(src::Dict, path_to_project::AbstractString)
     required = true
     config = ImportSource(src, "config", "config", "PhysiCell_settings.xml", "file", required)
     main = ImportSource(src, "main", "", "main.cpp", "file", required; input_folder_key = :custom_code)
@@ -42,14 +42,82 @@ function ImportSources(src::Dict)
 
     required = false
     rules = ImportSource(src, "rules", "config", "cell_rules.csv", "file", required; pcvct_name="base_rulesets.csv")
-    intracellular = ImportSource(src, "intracellular", "config", "intracellular.xml", "file", required)
+    intracellular = prepareIntracellularImport(src, config, path_to_project)
     ic_cell = ImportSource(src, "ic_cell", "config", "cells.csv", "file", required)
     ic_substrate = ImportSource(src, "ic_substrate", "config", "substrates.csv", "file", required)
     ic_ecm = ImportSource(src, "ic_ecm", "config", "ecm.csv", "file", required)
     ic_dc = ImportSource(src, "ic_dc", "config", "dcs.csv", "file", required)
     return ImportSources(config, main, makefile, custom_modules, rules, intracellular, ic_cell, ic_substrate, ic_ecm, ic_dc)
 end
 
+function prepareIntracellularImport(src::Dict, config::ImportSource, path_to_project::AbstractString)
+    if haskey(src, "intracellular") || isfile(joinpath(path_to_project, "config", "intracellular.xml"))
+        return ImportSource(src, "intracellular", "config", "intracellular.xml", "file", true)
+    end
+    #! now attempt to read the config file and assemble the intracellular file
+    path_to_xml = joinpath(path_to_project, config.path_from_project)
+    if !isfile(path_to_xml) #! if the config file is not found, then we cannot proceed with grabbing the intracellular data, just return the default
+        return ImportSource(src, "intracellular", "config", "intracellular.xml", "file", false)
+    end
+    xml_doc = openXML(path_to_xml)
+    cell_definitions_element = retrieveElement(xml_doc, ["cell_definitions"])
+    cell_type_to_components_dict = Dict{String,PhysiCellComponent}()
+    for cell_definition_element in child_elements(cell_definitions_element)
+        if name(cell_definition_element) != "cell_definition"
+            continue
+        end
+        cell_type = attribute(cell_definition_element, "name")
+        phenotype_element = find_element(cell_definition_element, "phenotype")
+        intracellular_element = find_element(phenotype_element, "intracellular")
+        if isnothing(intracellular_element)
+            continue
+        end
+        type = attribute(intracellular_element, "type")
+        if type ∉ ["roadrunner", "dfba"]
+            throw(ErrorException("pcvct does not yet support intracellular type $type."))
+        end
+        path_to_file = find_element(intracellular_element, "sbml_filename") |> content
+        temp_component = PhysiCellComponent(type, basename(path_to_file))
+        #! now we have to rely on the path to the file is correct relative to the parent directory of the config file (that should usually be the case)
+        path_to_src = joinpath(path_to_project, path_to_file)
+        path_to_dest = _create_component_dest_file_name(readlines(path_to_src), temp_component)
+        component = PhysiCellComponent(type, basename(path_to_dest))
+        if !isfile(path_to_dest)
+            cp(path_to_src, path_to_dest)
+        end
+
+        cell_type_to_components_dict[cell_type] = component
+    end
+
+    if isempty(cell_type_to_components_dict)
+        return ImportSource(src, "intracellular", "config", "intracellular.xml", "file", false)
+    end
+
+    intracellular_folder = assembleIntracellular!(cell_type_to_components_dict; name="temp_assembled_from_$(splitpath(path_to_project)[end])", skip_db_insert=true)
+    mv(joinpath(locationPath(:intracellular, intracellular_folder), "intracellular.xml"), joinpath(path_to_project, "config", "assembled_intracellular_for_import.xml"); force=true)
+    rm(locationPath(:intracellular, intracellular_folder); force=true, recursive=true)
+    
+    closeXML(xml_doc)
+    return ImportSource(src, "intracellular", "config", "assembled_intracellular_for_import.xml", "file", true; pcvct_name="intracellular.xml")
+end
+
+function _create_component_dest_file_name(src_lines::Vector{String}, component::PhysiCellComponent)
+    base_path = joinpath(data_dir, "components", component.path_from_components)
+    folder = dirname(base_path)
+    mkpath(folder)
+    base_filename, file_ext = basename(base_path) |> splitext
+    n = 0
+    path_to_dest = joinpath(folder, base_filename * file_ext)
+    while isfile(path_to_dest)
+        if src_lines == readlines(path_to_dest)
+            return path_to_dest
+        end
+        n += 1
+        path_to_dest = joinpath(folder, base_filename * "_$(n)" * file_ext)
+    end
+    return path_to_dest
+end
+
 mutable struct ImportDestFolder
     path_from_inputs::AbstractString
     created::Bool
@@ -79,7 +147,7 @@ function ImportDestFolders(path_to_project::AbstractString, dest::Dict)
 
     #! optional folders
     rules = ImportDestFolder(path_fn("rules", "rulesets_collections"), created, description)
-    intracellular = ImportDestFolder(path_fn("intracellular", "intracellular"), created, description)
+    intracellular = ImportDestFolder(path_fn("intracellular", "intracellulars"), created, description)
     ic_cell = ImportDestFolder(path_fn("ic_cell", joinpath("ics", "cells")), created, description)
     ic_substrate = ImportDestFolder(path_fn("ic_substrate", joinpath("ics", "substrates")), created, description)
     ic_ecm = ImportDestFolder(path_fn("ic_ecm", joinpath("ics", "ecms")), created, description)
@@ -101,7 +169,7 @@ The following keys are recognized: $(join(["`$fn`" for fn in fieldnames(ImportDe
 - `extreme_caution::Bool`: If true, will ask for confirmation before deleting any folders created during the import process. Care has been taken to ensure this is unnecessary. Provided for users who want to be extra cautious.
 """
 function importProject(path_to_project::AbstractString, src=Dict(), dest=Dict(); extreme_caution::Bool=false)
-    project_sources = ImportSources(src)
+    project_sources = ImportSources(src, path_to_project)
     import_dest_folders = ImportDestFolders(path_to_project, dest)
     success = resolveProjectSources!(project_sources, path_to_project)
     if success
@@ -299,9 +367,6 @@ function adaptMain(path_from_inputs::AbstractString)
         return true
     end
 
-    idx = findfirst(x->contains(x, "<fstream>"), lines)
-    insert!(lines, idx+1, "#include <getopt.h>")
-
     idx1 = findfirst(x->contains(x, "// load and parse settings file(s)"), lines)
     if isnothing(idx1)
         idx1 = findfirst(x->contains(x, "bool XML_status = false;"), lines)
@@ -314,19 +379,27 @@ function adaptMain(path_from_inputs::AbstractString)
             return false
         end
     end
-    idx2 = findfirst(x -> contains(x, "// copy config file to"), lines)
-    if isnothing(idx2)
-        idx2 = findfirst(x -> contains(x, "system(") && contains(x, "copy_command"), lines)
-        if isnothing(idx2)
-            msg = """
-            Could not identify where the copy command is in the main.cpp file.
-            Aborting the export process.
-            """
-            println(msg)
-            return false
-        end
-    end
-    deleteat!(lines, idx1:(idx2-1))
+    idx_not_xml_status = findfirst(x->contains(x, "!XML_status"), lines)
+    idx2 = idx_not_xml_status + findfirst(x -> contains(x, "}"), lines[idx_not_xml_status:end]) - 1
+    # idx2 = findfirst(x -> contains(x, "// copy config file to"), lines)
+    # if isnothing(idx2)
+    #     idx2 = findfirst(x -> contains(x, "system(") && contains(x, "copy_command"), lines)
+    #     if isnothing(idx2)
+    #         idx2 = findfirst(x -> contains(x, "// OpenMP setup"), lines)
+    #         if isnothing(idx2)
+    #             idx2 = findfirst(x -> contains(x, "omp_set_num_threads("), lines)
+    #             if isnothing(idx2)
+    #                 msg = """
+    #                 Could not identify where the copy command is in the main.cpp file.
+    #                 Aborting the export process.
+    #                 """
+    #                 println(msg)
+    #                 return false
+    #             end
+    #         end
+    #     end
+    # end
+    deleteat!(lines, idx1:idx2)
 
     parsing_block = """
         // read arguments

test/test-scripts/ImportTests.jl

@@ -86,5 +86,12 @@ success = importProject(path_to_project)
 
 # import the combined sbml project
 path_to_project = joinpath("PhysiCell", "sample_projects_intracellular", "combined", "template-combined")
+src = Dict("intracellular" => "sample_combined_sbmls.xml")
+success = importProject(path_to_project, src)
+@test success
+
+path_to_project = joinpath("PhysiCell", "sample_projects_intracellular", "ode", "ode_energy")
+success = importProject(path_to_project)
+@test success
 success = importProject(path_to_project)
 @test success

test/test-scripts/IntracellularTests.jl

@@ -19,5 +19,7 @@ macros_lines = pcvct.readMacrosFile(out.trial)
 @test "ADDON_ROADRUNNER" in macros_lines
 
 #! more test coverage
-intracellular = assembleIntracellular!(cell_to_components_dict; name="template-combined")
-@test intracellular == "template-combined" #! should not need to make a new folder, the assembly.toml file should show they match
+new_intracellular = assembleIntracellular!(cell_to_components_dict; name="template-combined")
+@test intracellular == new_intracellular #! should not need to make a new folder, the assembly.toml file should show they match
+
+export_folder = out |> getSimulationIDs |> first |> exportSimulation