Merge pull request #103 from drbergman/development

0.0.17

Author: drbergman

Project.toml

@@ -1,7 +1,7 @@
 name = "pcvct"
 uuid = "3c374bc7-7384-4f83-8ca0-87b8c727e6ff"
 authors = ["Daniel Bergman <[email protected]> and contributors"]
-version = "0.0.16"
+version = "0.0.17"
 
 [deps]
 AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f"

docs/src/misc/renaming.md

@@ -12,4 +12,6 @@ Here are the options brainstormed thus far:
   - PhysiBatch.jl
   - PhysiCellDB.jl
   - PhysiDB.jl (the clear name for make the database portion a separate package)
-  - PhysiCell.jl (kinda self-important to assume this will be all the PhysiCell stuff in Julia)
+  - PhysiCell.jl (kinda self-important to assume this will be all the PhysiCell stuff in Julia)
+
+I think I'm now leaning towards `ModelManager.jl` being the underlying framework that works across ABM (or other modeling paradigms) frameworks and then `PhysiCellModelManager.jl` is the PhysiCell-specific version.

src/VCTClasses.jl

@@ -284,88 +284,80 @@ struct Monad <: AbstractMonad
     id::Int #! integer uniquely identifying this monad
     n_replicates::Int #! (minimum) number of simulations belonging to this monad
     simulation_ids::Vector{Int} #! simulation ids belonging to this monad
-
     inputs::InputFolders #! contains the folder names for the simulations in this monad
-
     variation_id::VariationID
 
     function Monad(n_replicates::Int, inputs::InputFolders, variation_id::VariationID, use_previous::Bool)
+        feature_str = """
+        (\
+        physicell_version_id,\
+        $(join(locationIDNames(), ",")),\
+        $(join(locationVariationIDNames(), ","))\
+        ) \
+        """
+        value_str = """
+        (\
+        $(physicellVersionDBEntry()),\
+        $(join([inputs[loc].id for loc in project_locations.all], ",")),\
+        $(join([variation_id[loc] for loc in project_locations.varied],","))\
+        ) \
+        """
         monad_id = DBInterface.execute(db,
             """
-            INSERT OR IGNORE INTO monads (\
-            physicell_version_id,\
-            $(join(locationIDNames(), ",")),\
-            $(join(locationVariationIDNames(), ","))\
-            ) \
-            VALUES(\
-            $(physicellVersionDBEntry()),\
-            $(join([inputs[loc].id for loc in project_locations.all], ",")),\
-            $(join([variation_id[loc] for loc in project_locations.varied],","))\
-            ) \
-            RETURNING monad_id;
+            INSERT OR IGNORE INTO monads $feature_str VALUES $value_str RETURNING monad_id;
             """
         ) |> DataFrame |> x -> x.monad_id
         if isempty(monad_id)
             monad_id = constructSelectQuery(
                 "monads",
                 """
-                WHERE (\
-                physicell_version_id,\
-                $(join(locationIDNames(), ",")),\
-                $(join(locationVariationIDNames(), ","))\
-                )=\
-                (\
-                $(physicellVersionDBEntry()),\
-                $(join([inputs[loc].id for loc in project_locations.all], ",")),\
-                $(join([variation_id[loc] for loc in project_locations.varied],","))\
-                );\
-                """,
+                WHERE $feature_str=$value_str
+                """;
                 selection="monad_id"
             ) |> queryToDataFrame |> x -> x.monad_id[1] #! get the monad_id
         else
             monad_id = monad_id[1] #! get the monad_id
         end
         return Monad(monad_id, n_replicates, inputs, variation_id, use_previous)
     end
+
     function Monad(id::Int, n_replicates::Int, inputs::InputFolders, variation_id::VariationID, use_previous::Bool)
-        simulation_ids = use_previous ? readMonadSimulationIDs(id) : Int[]
-        num_sims_to_add = n_replicates - length(simulation_ids)
+        @assert id > 0 "id must be positive"
+        @assert n_replicates >= 0 "n_replicates must be non-negative"
+
+        previous_simulation_ids = readMonadSimulationIDs(id)
+        new_simulation_ids = Int[]
+        num_sims_to_add = n_replicates - (use_previous ? length(previous_simulation_ids) : 0)
         if num_sims_to_add > 0
             for _ = 1:num_sims_to_add
                 simulation = Simulation(inputs, variation_id) #! create a new simulation
-                push!(simulation_ids, simulation.id) #! add the simulation id to the monad
+                push!(new_simulation_ids, simulation.id)
             end
+            recordSimulationIDs(id, [previous_simulation_ids; new_simulation_ids]) #! record the simulation ids in a .csv file
         end
 
-        @assert id > 0 "id must be positive"
-        @assert n_replicates >= 0 "n_replicates must be non-negative"
-
-        #! this could be done when adding new simulation ids to save some fie I/O
-        #! doing it here just to make sure it is always up to date (and for consistency across classes)
-        recordSimulationIDs(id, simulation_ids) #! record the simulation ids in a .csv file
+        simulation_ids = use_previous ? [previous_simulation_ids; new_simulation_ids] : new_simulation_ids
 
         return new(id, n_replicates, simulation_ids, inputs, variation_id)
     end
 
 end
 
-function Monad(inputs::InputFolders, variation_id::VariationID; use_previous::Bool=true)
-    n_replicates = 0 #! not making a monad to run if not supplying the n_replicates info
+function Monad(inputs::InputFolders, variation_id::VariationID; use_previous::Bool=true, n_replicates::Int=0)
     Monad(n_replicates, inputs, variation_id, use_previous)
 end
 
-function getMonad(monad_id::Int, n_replicates::Int)
+function getMonad(monad_id::Int, n_replicates::Int, use_previous::Bool)
     df = constructSelectQuery("monads", "WHERE monad_id=$(monad_id);") |> queryToDataFrame
     if isempty(df)
         error("Monad $(monad_id) not in the database.")
     end
     inputs = [loc => df[1, locationIDName(loc)] for loc in project_locations.all] |> InputFolders
     variation_id = [loc => df[1, locationVarIDName(loc)] for loc in project_locations.varied] |> VariationID
-    use_previous = true
     return Monad(monad_id, n_replicates, inputs, variation_id, use_previous)
 end
 
-Monad(monad_id::Integer; n_replicates::Integer=0) = getMonad(monad_id, n_replicates)
+Monad(monad_id::Integer; n_replicates::Integer=0, use_previous::Bool=true) = getMonad(monad_id, n_replicates, use_previous)
 
 function Simulation(monad::Monad)
     return Simulation(monad.inputs, monad.variation_id)

src/VCTDatabase.jl

@@ -11,34 +11,25 @@ function initializeDatabase(path_to_database::String; auto_upgrade::Bool=false)
     is_new_db = !isfile(path_to_database)
     global db = SQLite.DB(path_to_database)
     SQLite.transaction(db, "EXCLUSIVE")
-    success = createSchema(is_new_db; auto_upgrade=auto_upgrade)
-    SQLite.commit(db)
-    if success
-        global initialized = true
-    end
-    return success
-end
-
-function initializeDatabase()
-    global db = SQLite.DB()
-    is_new_db = true
-    success = createSchema(is_new_db)
-    if success
+    try
+        createSchema(is_new_db; auto_upgrade=auto_upgrade)
+    catch e
+        SQLite.rollback(db)
+        println("Error initializing database: $e")
+        return false
+    else
+        SQLite.commit(db)
         global initialized = true
+        return true
     end
-    return success
 end
 
 function reinitializeDatabase()
     if !initialized
         return
     end
     global initialized = false
-    if db.file == ":memory:" #! if the database is in memory, re-initialize it
-        initializeDatabase()
-    else
-        initializeDatabase(db.file; auto_upgrade=true)
-    end
+    return initializeDatabase(db.file; auto_upgrade=true)
 end
 
 function createSchema(is_new_db::Bool; auto_upgrade::Bool=false)
@@ -197,7 +188,7 @@ function createPCVCTTable(table_name::String, schema::String; db::SQLite.DB=db)
         s *= "\n\tThis helps to normalize what the id names are for these entries."
         s *= "\n\tYour table $(table_name) does not end in 's'."
         s *= "\n\tSee retrieveID(location::Symbol, folder_name::String; db::SQLite.DB=db)."
-        error(s)
+        throw(ErrorException(s))
     end
     #! check that schema has PRIMARY KEY named as table_name without the s followed by _id
     id_name = locationIDName(Symbol(table_name[1:end-1]))
@@ -206,7 +197,7 @@ function createPCVCTTable(table_name::String, schema::String; db::SQLite.DB=db)
         s *= "\n\tThis helps to normalize what the id names are for these entries."
         s *= "\n\tYour schema $(schema) does not have \"$(id_name) INTEGER PRIMARY KEY\"."
         s *= "\n\tSee retrieveID(location::Symbol, folder_name::String; db::SQLite.DB=db)."
-        error(s)
+        throw(ErrorException(s))
     end
     SQLite.execute(db, "CREATE TABLE IF NOT EXISTS $(table_name) (
         $(schema)

src/VCTDeletion.jl

@@ -250,12 +250,8 @@ function resetDatabase(; force_reset::Bool=false, force_continue::Bool=false)
         rm_hpc_safe(joinpath(locationPath(:custom_code, custom_code_folder), baseToExecutable("project")); force=true)
     end
 
-    if db.file == ":memory:"
-        initializeDatabase()
-    else
-        rm_hpc_safe("$(db.file)"; force=true)
-        initializeDatabase("$(db.file)")
-    end
+    rm_hpc_safe("$(db.file)"; force=true)
+    initializeDatabase("$(db.file)")
     return nothing
 end
 

test/test-scripts/ClassesTests.jl

@@ -22,11 +22,14 @@ n_replicates = 1
 config_variation_ids = [1, 2]
 rulesets_collection_variation_ids = [1, 1]
 ic_cell_variation_ids = [0, 0]
+location_variation_ids = Dict{Symbol,Union{Integer,AbstractArray{<:Integer}}}(
+    :config => config_variation_ids,
+    :rulesets_collection => rulesets_collection_variation_ids,
+    :ic_cell => ic_cell_variation_ids
+)
 sampling = Sampling(inputs;
     n_replicates=n_replicates,
-    location_variation_ids=Dict{Symbol,Union{Integer,AbstractArray{<:Integer}}}(:config => config_variation_ids,
-                                                                                :rulesets_collection => rulesets_collection_variation_ids,
-                                                                                :ic_cell => ic_cell_variation_ids)
+    location_variation_ids=location_variation_ids
 )
 @test sampling isa Sampling
 

test/test-scripts/DatabaseTests.jl

@@ -23,11 +23,6 @@ mv(custom_code_src_folder, custom_code_dest_folder)
 mv(config_dest_folder, config_src_folder)
 mv(custom_code_dest_folder, custom_code_src_folder)
 
-# test memory db
-pcvct.initializeDatabase()
-pcvct.reinitializeDatabase()
-pcvct.initializeDatabase(joinpath(pcvct.data_dir, "vct.db"))
-
 # test bad table
 table_name_not_end_in_s = "test"
 @test_throws ErrorException pcvct.createPCVCTTable(table_name_not_end_in_s, "")
@@ -58,4 +53,13 @@ pcvct.variationIDs(:ic_ecm, Sampling(1))
 pcvct.variationsTable(:config, Sampling(1); remove_constants=true)
 pcvct.variationsTable(:rulesets_collection, Sampling(1); remove_constants=true)
 pcvct.variationsTable(:ic_cell, Sampling(1); remove_constants=true)
-pcvct.variationsTable(:ic_ecm, Sampling(1); remove_constants=true)
+pcvct.variationsTable(:ic_ecm, Sampling(1); remove_constants=true)
+
+# test bad folder
+path_to_bad_folder = joinpath(pcvct.data_dir, "inputs", "configs", "bad_folder")
+mkdir(path_to_bad_folder)
+
+@test pcvct.reinitializeDatabase() == false
+
+rm(path_to_bad_folder; force=true, recursive=true)
+@test pcvct.initializeDatabase(pcvct.db.file) == true

test/test-scripts/RunnerTests.jl

@@ -3,7 +3,11 @@ filename = split(filename, "/") |> last
 str = "TESTING WITH $(filename)"
 hashBorderPrint(str)
 
-hashBorderPrint("DATABASE SUCCESSFULLY INITIALIZED!")
+n_sims = length(Monad(1))
+monad = Monad(1; n_replicates=1, use_previous=false)
+run(monad)
+@test length(monad.simulation_ids) == 1 #! how many simulations were attached to this monad when run
+@test length(getSimulationIDs(monad)) == n_sims+1 #! how many simulations are stored in simulations.csv
 
 config_folder = "0_template"
 rulesets_collection_folder = "0_template"
@@ -33,13 +37,9 @@ out2 = run(inputs, discrete_variations)
 @test out.trial.inputs == out2.trial.inputs
 @test out.trial.variation_id == out2.trial.variation_id
 
-hashBorderPrint("SIMULATION SUCCESSFULLY RUN!")
-
 query = pcvct.constructSelectQuery("simulations", "WHERE simulation_id=1")
 df = pcvct.queryToDataFrame(query; is_row=true)
 
-hashBorderPrint("SIMULATION SUCCESSFULLY IN DB!")
-
 cell_type = "default"
 discrete_variations = DiscreteVariation[]
 xml_path = [pcvct.cyclePath(cell_type); "phase_durations"; "duration:index:0"]
@@ -53,13 +53,9 @@ push!(discrete_variations, DiscreteVariation(xml_path, 5.0))
 
 sampling = createTrial(simulation, discrete_variations; n_replicates=n_replicates)
 
-hashBorderPrint("SAMPLING SUCCESSFULLY CREATED!")
-
 out = run(sampling; force_recompile=false)
 @test out.n_success == length(sampling)
 
-hashBorderPrint("SAMPLING SUCCESSFULLY RUN!")
-
 out2 = run(simulation, discrete_variations; n_replicates=n_replicates, force_recompile=false)
 @test out2.trial isa Sampling
 @test out2.trial.id == sampling.id
@@ -69,8 +65,6 @@ out2 = run(simulation, discrete_variations; n_replicates=n_replicates, force_rec
 @test out2.n_scheduled == 0
 @test out2.n_success == 0
 
-hashBorderPrint("SUCCESSFULLY `run` WITHOUT CREATING SAMPLING!")
-
 n_simulations = length(sampling) #! number of simulations recorded (in .csvs) for this sampling
 n_expected_sims = n_replicates
 for discrete_variation in discrete_variations
@@ -83,15 +77,11 @@ n_variations = length(sampling.variation_ids)
 @test n_simulations == n_variations * n_replicates #! ...how many variation ids we recorded (number of rulesets_variations_ids must match variation_ids on construction of sampling)
 @test n_simulations == out.n_success #! ...how many simulations succeeded
 
-hashBorderPrint("SAMPLING SUCCESSFULLY IN CSVS!")
-
 out = run(sampling; force_recompile=false)
 
 # no new simulations should have been run
 @test out.n_success == 0
 
-hashBorderPrint("SUCCESSFULLY FOUND PREVIOUS SIMS!")
-
 trial = Trial([sampling])
 @test trial isa Trial
 
@@ -100,8 +90,6 @@ out = run(trial; force_recompile=false)
 # no new simulations should have been run
 @test out.n_success == 0
 
-hashBorderPrint("SUCCESSFULLY RAN TRIAL!")
-
 @test_warn "`runAbstractTrial` is deprecated. Use `run` instead." runAbstractTrial(trial; force_recompile=false)
 
 # run a sim that will produce an error