V2.1 Easier game!

Build-ParallelEnv-Target-OffPolicy-SingleStack-SideChannel-EndReward-Easy-V2.1
move remain time position after target state
easier game reward get, while watching or shoot at target area u get reward(attack mode), watching enemy u also get reward(Freemode)

Assets/Script/InGame/AgentWithGun.cs

@@ -10,6 +10,8 @@ using Unity.MLAgents;
 using Unity.MLAgents.Sensors;
 using Unity.MLAgents.Actuators;
 using System.Linq;
+using System.Drawing;
+using Color = UnityEngine.Color;
 
 /*TODO:
 √tag 攻击排他
@@ -235,7 +237,7 @@ public class AgentWithGun : Agent
         Vector3 point = new Vector3(thisCam.pixelWidth / 2, thisCam.pixelHeight / 2, 0);//发射位置
         Ray ray = thisCam.ScreenPointToRay(point);
         RaycastHit hit;
-        Debug.DrawRay(ray.origin, ray.direction * 100, Color.blue);
+        // Debug.DrawRay(ray.origin, ray.direction * 100, Color.blue);
         //按下鼠标左键
         if (shoot != 0 && gunReadyToggle == true)
         {
@@ -245,27 +247,78 @@ public class AgentWithGun : Agent
             {
                 if (hit.collider.tag != myTag && hit.collider.tag != "Wall")
                 {
+                    // kill enemy
                     GameObject gotHitObj = hit.transform.gameObject;//获取受到Ray撞击的对象
                     gotHitObj.GetComponent<states>().ReactToHit(Damage, gameObject);
                     shoot = 0;
                     return targetCon.hitEnemyReward(gotHitObj.transform.position);
                 }
             }
+            if (targetCon.targetTypeInt == (int)TargetController.Targets.Attack)
+            {
+                // while if attack mode
+                float targetDis = Vector3.Distance(blockContainer.thisBlock.transform.position, transform.position);
+                if (targetDis <= rayScript.viewDistance)
+                {
+                    // Debug.DrawRay(new Vector3(0,0,0), viewPoint, Color.red);
+                    if (Vector3.Distance(ray.origin + (ray.direction * targetDis), blockContainer.thisBlock.transform.position) <= blockContainer.thisBlock.firebasesAreaDiameter / 2)
+                    {
+                        // im shooting at target but didn't hit enemy
+                        // Debug.DrawRay(ray.origin, viewPoint-ray.origin, Color.blue);
+                        return paramContainer.shootTargetAreaReward;
+                    }
+                }
+            }
             shoot = 0;
             return paramContainer.shootReward;
         }
         else if (shoot != 0 && gunReadyToggle == false)
         {
+            // shoot without ready
             shoot = 0;
             return paramContainer.shootWithoutReadyReward;
         }
         else
         {
+            // do not shoot
             shoot = 0;
             return paramContainer.nonReward;
         }
     }
 
+    float facingReward()
+    {
+        float thisReward = 0;
+        Ray ray = thisCam.ScreenPointToRay(new Vector3(thisCam.pixelWidth / 2, thisCam.pixelHeight / 2, 0));
+        if (targetCon.targetTypeInt == (int)TargetController.Targets.Free)
+        {
+            //free mode
+            RaycastHit hit;
+            if (Physics.Raycast(ray, out hit, 100))
+            {
+                if (hit.collider.tag != myTag && hit.collider.tag != "Wall")
+                {
+                    thisReward = paramContainer.facingReward;
+                }
+            }
+        }
+        else if(targetCon.targetTypeInt == (int)TargetController.Targets.Attack)
+        {
+            float targetDis = Vector3.Distance(blockContainer.thisBlock.transform.position, transform.position);
+            if(targetDis <= rayScript.viewDistance)
+            {
+                // Debug.DrawRay(new Vector3(0,0,0), viewPoint, Color.red);
+                if (Vector3.Distance(ray.origin + (ray.direction * targetDis), blockContainer.thisBlock.transform.position) <= blockContainer.thisBlock.firebasesAreaDiameter / 2)
+                {
+                    // im watching target
+                    // Debug.DrawRay(ray.origin, viewPoint-ray.origin, Color.blue);
+                    thisReward = paramContainer.facingReward;
+                }
+            }
+        }
+        return thisReward;
+    }
+    
     // getEnemyNum 获取现场除了自己以外的敌人数量
     int getEnemyNum()
     {
@@ -322,6 +375,9 @@ public class AgentWithGun : Agent
         }
         // 射击动作reward判断
         epreward += ballistic() + sceneReward;
+        // facing reward
+        epreward += facingReward();
+        // spin penalty
         spinRecord.Add(mouseX);
         if (spinRecord.Count >= paramContainer.spinRecordMax)
         {
@@ -372,8 +428,8 @@ public class AgentWithGun : Agent
 
         //sensor.AddObservation(allEnemyNum); // 敌人数量 int
         sensor.AddObservation(targetStates);// targettype, target x,y,z, firebasesAreaDiameter
-        sensor.AddObservation(remainTime);
         sensor.AddObservation(targetCon.getInAreaState());
+        sensor.AddObservation(remainTime);
         sensor.AddObservation(myObserve); // 自机位置xyz+朝向 float[](4,1)
         sensor.AddObservation(rayTagResult); // 探测用RayTag结果 float[](raySensorNum,1)
         sensor.AddObservation(rayDisResult); // 探测用RayDis结果 float[](raySensorNum,1)

Assets/Script/InGame/ParameterContainer.cs

@@ -70,6 +70,11 @@ public class ParameterContainer : MonoBehaviour
     public float shootWithoutReadyReward = -0.15f;
     [Tooltip("Kill bonus reward stack to nothing happend reward")]
     public float killBonusReward = 0.0f;
+    [Tooltip("Facing to enemy's reward")]
+    public float facingReward = 2.0f;
+    [Tooltip("Shoot at target area but didn't hit enemy")]
+    public float shootTargetAreaReward = 5.0f;
+
     [Header("Penalty Rewards")]
     [Tooltip("Speed Penalty Reward")]
     public float speedPenalty = 0f;

Assets/Script/InGame/TargetController.cs

@@ -51,7 +51,7 @@ public class TargetController : MonoBehaviour
     private float freeProb;
     private float sceneSize;
     private float lastDistance;
-    private Vector3 targetPosition;
+    public Vector3 targetPosition;
     private Vector3 targetLocalPosition;
     private bool firstRewardFlag = true;
     private bool targetEnemySpawnFinish = false;
@@ -248,7 +248,8 @@ public class TargetController : MonoBehaviour
                 (nowDistance, inArea) = blockCont.getAgentTargetDistanceAndInside(AgentObj.transform.position);
                 envUICon.updateTargetGauge(blockCont.thisBlock.firebasesBelong, blockCont.thisBlock.belongMaxPoint);
                 float areaTargetReward = getDistanceReward(nowDistance, inArea);
-                if (blockCont.thisBlock.firebasesBelong >= blockCont.thisBlock.belongMaxPoint)
+                // if (blockCont.thisBlock.firebasesBelong >= blockCont.thisBlock.belongMaxPoint)
+                if(inArea != 0)
                 {
                     // win
                     // let the area belongs to me