Skip to content

Commit

Permalink
refactor: clean up GhostRLAgent logic and update training scene timers
Browse files Browse the repository at this point in the history
  • Loading branch information
@ddanakim0304
ddanakim0304 committed Dec 16, 2024
1 parent 75737c2 commit 39ece0b
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 133 deletions.
2 changes: 1 addition & 1 deletion Assets/ML-Agents/Timers/ghost_training_scene_timers.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"count":1,"self":68.5384832,"total":68.544885999999991,"children":{"InitializeActuators":{"count":1,"self":0.001093,"total":0.001093,"children":null},"InitializeSensors":{"count":1,"self":0.001029,"total":0.001029,"children":null},"AgentSendState":{"count":377,"self":0.0013189999999999999,"total":0.0013189999999999999,"children":null},"DecideAction":{"count":377,"self":0.002188,"total":0.002188,"children":null},"AgentAct":{"count":377,"self":0.000368,"total":0.000368,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1734361195","unity_version":"2022.3.38f1","command_line_arguments":"\/Applications\/Unity\/Hub\/Editor\/2022.3.38f1\/Unity.app\/Contents\/MacOS\/Unity -projectpath \/Users\/yeinkim\/Documents\/GameDev\/Pacman AI -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-yeinkim -hubSessionId b0fd1799-a053-4c7c-85e2-97e4b41fac50 -accessToken _WhrwS0LAEBbPLbRjNtUHRQxC-jyC8GeFEngL1pogRo004f","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"ghost_training_scene","end_time_seconds":"1734361263"}}
{"count":1,"self":10.900432,"total":10.916991999999999,"children":{"InitializeActuators":{"count":1,"self":0.001166,"total":0.001166,"children":null},"InitializeSensors":{"count":1,"self":0.001036,"total":0.001036,"children":null},"AgentSendState":{"count":473,"self":0.001366,"total":0.001366,"children":null},"DecideAction":{"count":473,"self":0.002156,"total":0.002156,"children":null},"AgentAct":{"count":473,"self":0.009646,"total":0.009646,"children":null}},"gauges":{"GhostRLAgent.CumulativeReward":{"count":1,"max":0,"min":0,"runningAverage":0,"value":0,"weightedAverage":0}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1734370166","unity_version":"2022.3.38f1","command_line_arguments":"\/Applications\/Unity\/Hub\/Editor\/2022.3.38f1\/Unity.app\/Contents\/MacOS\/Unity -projectpath \/Users\/yeinkim\/Documents\/GameDev\/Pacman AI -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"3.0.0","scene_name":"ghost_training_scene","end_time_seconds":"1734370177"}}
135 changes: 54 additions & 81 deletions Assets/Scenes/ghost_training_scene.unity
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,53 @@ Transform:
m_Children: []
m_Father: {fileID: 635587384}
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &586083504
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 586083506}
- component: {fileID: 586083505}
m_Layer: 0
m_Name: GameManager
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!114 &586083505
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 586083504}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 589bd1e63045b4e3997d03c525507826, type: 3}
m_Name:
m_EditorClassIdentifier:
ghosts: []
pacman: {fileID: 0}
pellets: {fileID: 813467038}
--- !u!4 &586083506
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 586083504}
serializedVersion: 2
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 1.6244671, y: 3.2080235, z: 0.018421387}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &591046346 stripped
GameObject:
m_CorrespondingSourceObject: {fileID: 7807851060747111036, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
Expand Down Expand Up @@ -1131,7 +1178,6 @@ GameObject:
- component: {fileID: 656308240}
- component: {fileID: 656308239}
- component: {fileID: 656308238}
- component: {fileID: 656308243}
m_Layer: 6
m_Name: Pacman
m_TagString: Player
Expand Down Expand Up @@ -1268,18 +1314,6 @@ Transform:
m_Children: []
m_Father: {fileID: 0}
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &656308243
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 656308237}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: b08b1ca5b32f34f77903d997e7a4b867, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &710022715
GameObject:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -3959,57 +3993,6 @@ Tilemap:
e31: 0
e32: 0
e33: 1
--- !u!1 &1028602778
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 1028602779}
- component: {fileID: 1028602780}
m_Layer: 0
m_Name: GameManager
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &1028602779
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1028602778}
serializedVersion: 2
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -0.64744174, y: 6.375536, z: -0.10777246}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &1028602780
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1028602778}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 589bd1e63045b4e3997d03c525507826, type: 3}
m_Name:
m_EditorClassIdentifier:
ghosts:
- {fileID: 0}
- {fileID: 0}
- {fileID: 0}
- {fileID: 1456563231}
pacman: {fileID: 656308243}
pellets: {fileID: 813467038}
--- !u!1 &1051761051
GameObject:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -9353,17 +9336,6 @@ TilemapCollider2D:
m_MaximumTileChangeCount: 1000
m_ExtrusionFactor: 0
m_UseDelaunayMesh: 0
--- !u!114 &1456563231 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 8865373448656147578, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
m_PrefabInstance: {fileID: 2077738158}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 591046346}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8d1191079784c40359fce5829f6c6274, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &1456563232
MonoBehaviour:
m_ObjectHideFlags: 0
Expand All @@ -9379,9 +9351,9 @@ MonoBehaviour:
agentParameters:
maxStep: 0
hasUpgradedFromAgentParameters: 1
MaxStep: 0
MaxStep: 100
pacman: {fileID: 656308241}
iterationText: {fileID: 1051761053}
iterationText: {fileID: 0}
--- !u!114 &1456563233
MonoBehaviour:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -9425,7 +9397,7 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: e9b63d5661d3e42e194b890d2914a905, type: 3}
m_Name:
m_EditorClassIdentifier:
duration: 20
duration: 1000
rlAgent: {fileID: 1456563232}
--- !u!1 &1730778446
GameObject:
Expand Down Expand Up @@ -9697,11 +9669,11 @@ PrefabInstance:
objectReference: {fileID: 1822597962}
- target: {fileID: 3877934034716477805, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
propertyPath: duration
value: 5
value: 0
objectReference: {fileID: 0}
- target: {fileID: 4892246748427573217, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
propertyPath: speed
value: 100
value: 20
objectReference: {fileID: 0}
- target: {fileID: 7807851060747111036, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
propertyPath: m_Name
Expand Down Expand Up @@ -9765,6 +9737,7 @@ PrefabInstance:
objectReference: {fileID: 0}
m_RemovedComponents:
- {fileID: 1897629771422341733, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
- {fileID: 3877934034716477805, guid: 66ce666aac68d45a684d73516a9c9cb7, type: 3}
m_RemovedGameObjects: []
m_AddedGameObjects: []
m_AddedComponents:
Expand All @@ -9784,8 +9757,8 @@ SceneRoots:
m_Roots:
- {fileID: 465062962}
- {fileID: 2046344527}
- {fileID: 1028602779}
- {fileID: 656308241}
- {fileID: 2077738158}
- {fileID: 1919839060}
- {fileID: 1730778449}
- {fileID: 586083506}
2 changes: 1 addition & 1 deletion Assets/Scripts/GhostChase(RL).cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ private void OnTriggerEnter2D(Collider2D other)
}
}
}
}
}
59 changes: 10 additions & 49 deletions Assets/Scripts/GhostRLAgent.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,12 @@

public class GhostRLAgent : Agent
{
public Transform pacman;
private Node currentNode;
public Transform pacman;

private Vector2 chosenDirection;
private int stepCount;
private const int maxStepsPerEpisode = 100;

// Display training iteration in UI
[SerializeField] private TMPro.TextMeshProUGUI iterationText;
[SerializeField] private TMPro.TextMeshProUGUI iterationText; // UI text component
private int trainingIteration = 0;

public override void OnEpisodeBegin()
Expand All @@ -26,16 +23,8 @@ public override void OnEpisodeBegin()
{
iterationText.text = $"Training Iteration: {trainingIteration}";
}
// Reset ghost to its initial position
transform.position = transform.position;

// Set Pac-Man's position (fixed for this phase)
pacman.position = new Vector3(5, 5, 0);

// Reset variables
chosenDirection = Vector2.zero;
currentNode = null;
stepCount = 0;
}

public override void CollectObservations(VectorSensor sensor)
Expand All @@ -44,30 +33,19 @@ public override void CollectObservations(VectorSensor sensor)
Vector2 relativePosition = (Vector2)pacman.position - (Vector2)transform.position;
sensor.AddObservation(relativePosition.normalized);

// Observe available directions from the current node
if (currentNode != null)
// Observe available directions at the current node
foreach (Vector2 dir in currentNode.availableDirections)
{
foreach (Vector2 dir in currentNode.availableDirections)
{
sensor.AddObservation(dir.normalized);
}
}
else
{
// If no current node, add zero observations for consistency
for (int i = 0; i < 4; i++)
{
sensor.AddObservation(Vector2.zero);
}
sensor.AddObservation(dir.normalized);
}
}

public override void OnActionReceived(ActionBuffers actions)
{
// Get the action (direction index) from the RL model
// Get action index
int action = actions.DiscreteActions[0];

// Map action to a direction and move the ghost
// Map action to a direction
if (currentNode != null && action < currentNode.availableDirections.Count)
{
chosenDirection = currentNode.availableDirections[action];
Expand All @@ -77,32 +55,15 @@ public override void OnActionReceived(ActionBuffers actions)
chosenDirection = Vector2.zero;
}

// Move ghost
if (chosenDirection != Vector2.zero)
{
transform.position += (Vector3)chosenDirection;
}

// Reward and termination logic
stepCount++;
// Reward logic
float distanceToPacman = Vector2.Distance(transform.position, pacman.position);
AddReward(-distanceToPacman * 0.01f); // Slight penalty for distance

// Reward for moving closer
AddReward(-distanceToPacman * 0.01f);

// End episode if ghost reaches Pac-Man
if (distanceToPacman < 1.0f)
{
AddReward(1.0f); // Big reward for catching Pac-Man
EndEpisode();
}

// End episode if maximum steps are reached
if (stepCount >= maxStepsPerEpisode)
{
AddReward(-1.0f); // Penalty for failing to reach Pac-Man
EndEpisode();
}
}

public void SetCurrentNode(Node node)
Expand All @@ -114,4 +75,4 @@ public Vector2 GetChosenDirection()
{
return chosenDirection;
}
}
}
1 change: 0 additions & 1 deletion Assets/Scripts/GhostScatter.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
using UnityEditor.Experimental.GraphView;
using UnityEngine;

public class GhostScatter : GhostBehavior
Expand Down
Binary file added mono_crash.mem.9670.1.blob
View file
Binary file not shown.

0 comments on commit 39ece0b

Please sign in to comment.