update examples

README.md

@@ -90,3 +90,7 @@ The tables above show nodes occupience by job at a given time.
 
 For actual work Visual Studio Code, Jupyter Notebook and Pluto Notebook are suggested.
 
+## Exmaples
+
+For examples see [examples](examples/) 
+

examples/README.md

@@ -0,0 +1,10 @@
+# HPCMod Examples
+
+Below we will point to jupyter notebooks example, for plain julia files see [plain_jl](plain_jl/).
+
+
+## Simple Ten Node Cluster: Jobs Traces Replay
+
+
+* [Job Trace Replay](jupyter/simple_job_trace_replay.ipynb) 
+* [Compute Tasks](jupyter/simple_models.ipynb)

examples/simple_job_trace_replay.ipynb → examples/jupyter/simple_job_trace_replay.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# HPCMod Example: Jobs Traces Replay on Ten Node Cluster\n",
+    "# Simple Ten Node Cluster: Jobs Traces Replay\n",
     "\n",
     "Simple Ten Node Cluster is homoginious cluster containing only ten nodes. The scheduler it simple First Come First Served with a backfiller.\n",
     "\n",

examples/plain_jl/simple2.jl

@@ -0,0 +1,61 @@
+# HPCMod Example
+# Check the backfiller effect
+# 
+
+using Agents
+using DataFrames
+using Tidier
+using Random
+using HPCMod
+
+function run_sim(;scheduler_backfill=true, rng=Random.Xoshiro(123))
+    sim = Simulation(;rng)
+
+    add_resource!(
+        sim; 
+        nodes=10,
+        max_nodes_per_job=4,
+        max_time_per_job=24*3,
+        scheduler_backfill)
+
+    for n in 1:18
+        user = User(
+            sim;
+            max_concurrent_tasks=2
+            )
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=4, walltime_prefered=50)
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=4, walltime_prefered=50)
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=4, walltime_prefered=50)
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=4, walltime_prefered=50)
+    end
+
+    for n in 1:2
+        user = User(
+            sim;
+            max_concurrent_tasks=2
+            )
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=2, walltime_prefered=50)
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=2, walltime_prefered=50)
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=2, walltime_prefered=50)
+            CompTask(sim, user.id; nodetime=100, nodes_prefered=2, walltime_prefered=50)
+    end
+
+    run!(sim; nsteps=2000)
+
+    timeunits_per_day = sim.timeunits_per_day
+    mdf = @chain sim.mdf begin
+        @filter(used_nodes > 0)
+        @summarise begin
+            mean_used_nodes = mean(used_nodes)
+            sd_used_nodes = std(used_nodes)
+            max_time = maximum(time)
+        end
+        @mutate max_days = max_time / !!timeunits_per_day 
+    end
+    println(mdf)
+    return sim
+end
+
+sim_bf = run_sim(;scheduler_backfill=true);
+sim_nobf = run_sim(;scheduler_backfill=false);
+

examples/simple_models.jl → examples/plain_jl/simple_models.jl

@@ -13,9 +13,16 @@ Simulate 10-Node Cluster
 * `adaptive_fourth_user` - should the 4th user use AdaptiveFactor strategy
 * `thinktime_gamma` - should a gamma distribution used for think time between jobs of same task
 """
-function simulate(;adaptive_fourth_user=false, thinktime_gamma=false, max_time_per_job=24*3)
+function simulate(
+	;
+	adaptive_fourth_user=false, 
+	thinktime_gamma=false, 
+	max_time_per_job=24*3, 
+	all_adaptive=false,
+	rng=Random.Xoshiro(123),
+	arrive_same_time=true)
 	# Init simulation, seed a random generator
-	sim = Simulation(;rng=Random.Xoshiro(123))
+	sim = Simulation(;rng=rng)
 	sim.workload_done_check_freq = 1
 	# Add HPC resource
 	add_resource!(
@@ -34,11 +41,20 @@ function simulate(;adaptive_fourth_user=false, thinktime_gamma=false, max_time_p
 		if thinktime_gamma
 	    	user.thinktime_generator=generate_thinktime_gamma
 		end
+
+		adaptive_user = false
+		if adaptive_fourth_user && user_id==4
+			adaptive_user = true
+		end
+
+		if all_adaptive
+			adaptive_user = true
+		end
 
 	    for m_task_id in 1:1
 	        CompTask(sim, user.id;
-	            task_split_schema = adaptive_fourth_user ? ( user_id==4 ? AdaptiveFactor : UserPreferred ) : UserPreferred,
-	            submit_time=user_id+m_task_id,
+	            task_split_schema = adaptive_user ? AdaptiveFactor : UserPreferred,
+	            submit_time= arrive_same_time ? 1 : user_id+m_task_id,
 	            nodetime=140, nodes_prefered=4, walltime_prefered=48)
 	    end
 	end
@@ -52,6 +68,8 @@ base = simulate(adaptive_fourth_user=false, thinktime_gamma=false);
 base24 = simulate(adaptive_fourth_user=false, thinktime_gamma=true, max_time_per_job=24);
 adaptive = simulate(adaptive_fourth_user=true, thinktime_gamma=false);
 adaptive_gamma = simulate(adaptive_fourth_user=true, thinktime_gamma=true);
+adaptive_gamma2 = simulate(adaptive_fourth_user=true, thinktime_gamma=true,rng=Random.Xoshiro(131));
+adaptive_all = simulate(all_adaptive=true, thinktime_gamma=true);
 
 #plotly(ticks=:native)
 #"A. Prefered User and no Think Time"
@@ -67,8 +85,20 @@ p3=plot_node_util(adaptive, ticks_step=0.5, annotation_pointsize=annotation_poin
 title!("C.", titleloc=:left)
 p4=plot_node_util(adaptive_gamma,ticks_step=0.5, annotation_pointsize=annotation_pointsize)
 title!("D.", titleloc=:left)
+p5=plot_node_util(adaptive_gamma2,ticks_step=0.5, annotation_pointsize=annotation_pointsize)
+title!("E.", titleloc=:left)
+p6=plot_node_util(adaptive_all,ticks_step=0.5, annotation_pointsize=annotation_pointsize)
+title!("F.", titleloc=:left)
 
-plot(p1,p2,p3,p4, layout = (2, 2),size=(600,600), colorbar = false, left_margin = 5Plots.mm, bottom_margin = 5Plots.mm)
+plot(p1,p2,p3,p4,p5,p6,layout = (2, 3),size=(800,600), colorbar = false, left_margin = 5Plots.mm, bottom_margin = 5Plots.mm)
+
+base_arrive_same_time = simulate(arrive_same_time=true, max_time_per_job=24);
+adaptive_all_arrive_same_time = simulate(all_adaptive=true, arrive_same_time=true, thinktime_gamma=true, max_time_per_job=24);
+
+p1=plot_node_util(base_arrive_same_time, ticks_step=0.5, annotation_pointsize=annotation_pointsize)
+title!("A.", titleloc=:left)
+p2=plot_node_util(adaptive_all_arrive_same_time, ticks_step=0.5, annotation_pointsize=annotation_pointsize)
+title!("B.", titleloc=:left)
 
 #savefig("node_util2.svg")
 #savefig("node_util2.pdf")

test/scheduler/scheduler_test2.jl

@@ -26,4 +26,4 @@ sim = jobs_replay_on_resource(DataFrame([
             104 6 101 2 4], [
             "job_id", "submit_time", "user_id", "nodes", "walltime"
         ]), ; nodes=4, scheduler_backfill=true, workload_done_check_freq=1);
-show(stdout, "text/plain", Matrix(sim.resource.stats.node_occupancy_by_job))
+show(stdout, "text/plain", Matrix(sim.resource.stats.node_occupancy_by_job))

test/testset1.jl

@@ -37,7 +37,7 @@ end
 
 
 @testset "Examples: simple1 " begin
-    include("../examples/simple1.jl")
+    include("../examples/plain_jl/simple1.jl")
 
     # check that it at lest produce something
     @test nrow(sim.mdf) > 50
@@ -47,6 +47,20 @@ end
     @test sum(sim.mdf.jobs_done) > 0
 end
 
+@testset "Examples: simple2 " begin
+    include("../examples/plain_jl/simple2.jl")
+
+    # check that it at lest produce something
+    @test nrow(sim_nobf.mdf)==2001
+    @test nrow(sim_bf.mdf)==2001
+end
+
+@testset "Examples: simple_comp_tasks " begin
+    include("../examples/plain_jl/simple_comp_tasks.jl")
+
+    # check that it at lest produce something
+    @test abmtime(sim1.model)==19
+end
 
 @testset "DateTime Conversion " begin
     sim = Simulation()