Koha9/Aimbot-ParallelEnv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V2.1 Add spin penalty and reward adjust

Browse Source 
Build-ParallelEnv-Target-OffPolicy-SingleStack-SideChannel-EndReward-Easy-V2.1
add spin penalty while agent keep spin will give a penalty reward.
lower Go target in area reward.
This commit is contained in:
Koha9
2022-12-04 08:40:23 +09:00
parent 0d60c857b7
commit fa90ac3bb6
6 changed files with 185 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Assets/ML-Agents/Timers/InGame_timers.json
+1 -1
View File
@@ -1 +1 @@
{"count":1,"self":29.5144064,"total":31.109803999999997,"children":{"InitializeActuators":{"count":16,"self":0.0020061,"total":0.0020061,"children":null},"InitializeSensors":{"count":16,"self":0.0015064,"total":0.0015064,"children":null},"AgentSendState":{"count":1300,"self":0.032629399999999996,"total":1.3793506,"children":{"CollectObservations":{"count":10400,"self":1.3266885,"total":1.3266885,"children":null},"WriteActionMask":{"count":10400,"self":0.0055122,"total":0.0055122,"children":null},"RequestDecision":{"count":10400,"self":0.014520499999999999,"total":0.014520499999999999,"children":null}}},"DecideAction":{"count":1300,"self":0.0246036,"total":0.0246036,"children":null},"AgentAct":{"count":1300,"self":0.18743069999999998,"total":0.18743069999999998,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1670056714","unity_version":"2021.3.14f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2021.3.14f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-ParallelEnv -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-UCUNI -hubSessionId f236fc50-7039-11ed-847b-693e2468b0ed -accessToken 9ZtagkYCGQGJbz1h79OWqFnBSy12hJOOJ_slXKTD6NI00ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"InGame","end_time_seconds":"1670056744"}}
{"count":1,"self":10.5192528,"total":10.849220899999999,"children":{"InitializeActuators":{"count":16,"self":0.0015046,"total":0.0015046,"children":null},"InitializeSensors":{"count":16,"self":0.0015007,"total":0.0015007,"children":null},"AgentSendState":{"count":286,"self":0.0080161,"total":0.2672959,"children":{"CollectObservations":{"count":2288,"self":0.2547756,"total":0.2547756,"children":null},"WriteActionMask":{"count":2288,"self":0.0010035,"total":0.0010035,"children":null},"RequestDecision":{"count":2288,"self":0.0035007,"total":0.0035007,"children":null}}},"DecideAction":{"count":286,"self":0.0080269,"total":0.0080269,"children":null},"AgentAct":{"count":286,"self":0.0511388,"total":0.0511388,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1670110662","unity_version":"2021.3.14f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2021.3.14f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-ParallelEnv -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-UCUNI -hubSessionId e96bd570-7311-11ed-acd6-8dc2264f2f57 -accessToken MoIZzq15aBLxocQuZONQNXLroHJT33mJvl_8kIgeC9I00ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"InGame","end_time_seconds":"1670110673"}}
Assets/Scenes/InGame.unity
+72 -56
View File
@@ -2744,7 +2744,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -8228,13 +8228,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -8247,8 +8249,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -9550,13 +9552,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -9569,8 +9573,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -17322,7 +17326,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -19592,13 +19596,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -19611,8 +19617,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -25421,7 +25427,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -30251,7 +30257,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -30866,13 +30872,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -30885,8 +30893,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -33261,7 +33269,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -34532,7 +34540,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -35432,7 +35440,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -38454,13 +38462,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -38473,8 +38483,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -42051,7 +42061,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 38
VectorObservationSize: 39
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 1
@@ -42937,13 +42947,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -42956,8 +42968,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -51336,13 +51348,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -51355,8 +51369,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
@@ -53598,13 +53612,15 @@ MonoBehaviour:
timeLimit: 30
lockCameraX: 0
lockCameraY: 1
chartOn: 0
chartOn: 1
spinRecordMax: 40
spinPenaltyThreshold: 10
hitTargetRewardDefault: 100
killTargetEnemyRewardDefault: 100
inAreaRewardDefault: 10
inAreaRewardDefault: 2
freeTimeBonusPerSec: 1
targetTimeBonusPerSec: 2
areaTimeBonusPerSec: 1
targetTimeBonusPerSec: 1
areaTimeBonusPerSec: 0.2
distanceReward: 20
goWinRewardDefault: 999
attackWinRewardDefault: 999
@@ -53617,8 +53633,8 @@ MonoBehaviour:
killReward: 5
shootWithoutReadyReward: -0.15
killBonusReward: 0
speedPanalty: 0
viewPanalty: 0
speedPenalty: 0
spinPenalty: 0.04
hitTargetReward: 60
killTargetEnemyReward: 80
inAreaReward: 5
Assets/Script/InGame/AgentWithGun.cs
+14 -2
View File
@@ -38,6 +38,7 @@ public class AgentWithGun : Agent
public float yRotation = 0.1f;//定义一个浮点类型的量，记录‘围绕’X轴旋转的角度
[Header("Env")]
private List<float> spinRecord = new List<float>();
private bool lockMouse;
private float Damage;
private float fireRate;
@@ -302,7 +303,7 @@ public class AgentWithGun : Agent
// ------------Reward--------------
// rewardCalculate 计算本动作的Reward
public float rewardCalculate(float sceneReward)
public float rewardCalculate(float sceneReward,float mouseX)
{
float epreward = 0f;
// 击杀reward判断
@@ -321,6 +322,16 @@ public class AgentWithGun : Agent
}
// 射击动作reward判断
epreward += ballistic() + sceneReward;
spinRecord.Add(mouseX);
if (spinRecord.Count >= paramContainer.spinRecordMax)
{
spinRecord.RemoveAt(0);
}
float spinPenaltyReward = Math.Abs(spinRecord.ToArray().Sum() * paramContainer.spinPenalty);
if(spinPenaltyReward >= paramContainer.spinPenaltyThreshold)
{
epreward -= spinPenaltyReward;
}
return epreward;
}
@@ -362,6 +373,7 @@ public class AgentWithGun : Agent
//sensor.AddObservation(allEnemyNum); // 敌人数量 int
sensor.AddObservation(targetStates);// targettype, target x,y,z, firebasesAreaDiameter
sensor.AddObservation(remainTime);
sensor.AddObservation(targetCon.getInAreaState());
sensor.AddObservation(myObserve); // 自机位置xyz+朝向 float[](4,1)
sensor.AddObservation(rayTagResult); // 探测用RayTag结果 float[](raySensorNum,1)
sensor.AddObservation(rayDisResult); // 探测用RayDis结果 float[](raySensorNum,1)
@@ -393,7 +405,7 @@ public class AgentWithGun : Agent
float sceneReward = 0f;
float endReward = 0f;
(finishedState, sceneReward, endReward) = targetCon.checkOverAndRewards();
float thisRoundReward = rewardCalculate(sceneReward+ endReward);
float thisRoundReward = rewardCalculate(sceneReward+ endReward,Mouse_X);
if (paramContainer.chartOn)
{
EnvUICon.updateChart(thisRoundReward);
Assets/Script/InGame/ParameterContainer.cs
+4 -2
View File
@@ -22,6 +22,8 @@ public class ParameterContainer : MonoBehaviour
public bool lockCameraX = false;
public bool lockCameraY = true;
public bool chartOn = false;
public int spinRecordMax = 20;
public float spinPenaltyThreshold = 50;
[Header("Dynamic Defaut Rewards")]
@@ -70,9 +72,9 @@ public class ParameterContainer : MonoBehaviour
public float killBonusReward = 0.0f;
[Header("Penalty Rewards")]
[Tooltip("Speed Penalty Reward")]
public float speedPanalty = 0f;
public float speedPenalty = 0f;
[Tooltip("view Panalty Reward")]
public float viewPanalty = 0f;
public float spinPenalty = 0f;
[Header("Dynamic Rewards")]
[Tooltip("Free mode Hit Enemy reward")]
Assets/Script/InGame/TargetController.cs
+25 -4
View File
@@ -1,6 +1,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Xml.Serialization;
using UnityEngine;
using Random = UnityEngine.Random;
@@ -46,6 +47,7 @@ public class TargetController : MonoBehaviour
[System.NonSerialized] public int targetNum = 0;
private Dictionary<int, float[]> oneHotRarget = new Dictionary<int, float[]>();
private int inArea = 0;
private float freeProb;
private float sceneSize;
private float lastDistance;
@@ -238,7 +240,6 @@ public class TargetController : MonoBehaviour
int endTypeInt = 0;
float thisReward = 0;
float endReward = 0;
int inArea = 0;
float nowDistance = 0f;
switch (targetTypeInt)
{
@@ -246,7 +247,7 @@ public class TargetController : MonoBehaviour
// goto
(nowDistance, inArea) = blockCont.getAgentTargetDistanceAndInside(AgentObj.transform.position);
envUICon.updateTargetGauge(blockCont.thisBlock.firebasesBelong, blockCont.thisBlock.belongMaxPoint);
float areaTargetReward = (paramCon.inAreaReward * inArea) + getDistanceReward(nowDistance);
float areaTargetReward = getDistanceReward(nowDistance, inArea);
if (blockCont.thisBlock.firebasesBelong >= blockCont.thisBlock.belongMaxPoint)
{
// win
@@ -363,7 +364,7 @@ public class TargetController : MonoBehaviour
}
// caulculate sceneReward if close to target then get great reward
public float getDistanceReward(float nowDistance)
public float getDistanceReward(float nowDistance,int inarea)
{
if (firstRewardFlag)
{
@@ -371,7 +372,14 @@ public class TargetController : MonoBehaviour
firstRewardFlag = false;
}
float thisSceneReward = 0f;
thisSceneReward = paramCon.distanceReward * (lastDistance - nowDistance);
if (inarea != 0)
{
thisSceneReward = paramCon.inAreaReward;
}
else
{
thisSceneReward = paramCon.distanceReward * (lastDistance - nowDistance);
}
lastDistance = nowDistance;
return thisSceneReward;
}
@@ -438,4 +446,17 @@ public class TargetController : MonoBehaviour
}
return thisHitReward;
}
// get in area state
public int getInAreaState()
{
if(targetTypeInt == (int)Targets.Go)
{
return inArea;
}
else
{
return 0;
}
}
}