Target & dynamic reward System V1.0

太多了...总之能用了，就这样吧，也懒得用英文写了(;´Д`) ParameterContainer用于管理所有的Reward和Parameter. TargetController用于生成目标，然后再让SceneBlockContainer实际生成目标块，并且兼顾reward计算功能和目标观察结果的获取. EnemyContainer用于生成和删除敌人. SceneBlockContainer用于生成和删除目标块. States用于管理HP. SceneBlock用于管理目标块的...一堆东西，比如目标大小，目标区内人数等和目标所属状态等.
2022-11-29 07:54:08 +09:00
parent 6e32da8b64
commit 25eac00c53
84 changed files with 42271 additions and 10510 deletions
@@ -11,25 +11,22 @@ using Unity.MLAgents.Sensors;
 using Unity.MLAgents.Actuators;
 using System.Linq;

-/*主要ML-Agent控制*/
+/*TODO:
+√tag 攻击排他
+√通用HP 系统
+环境tag修正？
+以tag重置环境修正
+Agent死亡时待机处理*/

 public class AgentWithGun : Agent
 {
    public GameObject ParameterContainerObj;
-    public GameObject EnviromentObj;
+    public GameObject EnvironmentObj;
    public GameObject EnemyContainerObj;
-    public GameObject thisAgentObj;
+    public GameObject SceneBlockContainerObj;
    public GameObject EnvironmentUIControlObj;
-    public Transform thisAgent;
+    public GameObject TargetControllerObj;
    public Camera thisCam;
-    public CharacterController PlayerController;
-    public GameObject enemyPrefab;
-    public GameObject edgeUp;
-    public GameObject edgeDown;
-    public GameObject edgeLeft;
-    public GameObject edgeRight;
-    public GameObject edgeAgent_Enemy;
-

    [Header("GetAxis() Simulate")]
    public float MoveSpeed = 2.0f;
@@ -45,79 +42,50 @@ public class AgentWithGun : Agent
    private float Damage;
    private float fireRate;
    private int enemyNum;
-    private int timeLimit;
    private bool lockCameraX;
    private bool lockCameraY;
-    //public Vector3 startPosition = new Vector3(9, 1, 18);

-    private float startTime = 0;
+    // environment
    private int shoot = 0;
    private float lastShootTime = 0.0f;
    private int nowEnemyNum = 0;
    private int enemyKillCount = 0;
+    private Vector3 killEnemyPosition;
    private int step = 0;
    private int EP = 0;
    public bool defaultTPCamera = true;
    private bool gunReadyToggle = true;
+    private string myTag = "";
+    // scripts
    private RaySensors rayScript;
-    private EnviromentUIControl EnvUICon;
-    private parameterContainer paramContainer;
+    private CharacterController PlayerController;
+    private EnvironmentUIControl EnvUICon;
+    private ParameterContainer paramContainer;
+    private SceneBlockContainer blockContainer;
+    private EnemyContainer eneContainer;
+    private TargetController targetCon;

-    [System.NonSerialized] public float minEnemyAreaX;
-    [System.NonSerialized] public float maxEnemyAreaX;
-    [System.NonSerialized] public float minEnemyAreaZ;
-    [System.NonSerialized] public float maxEnemyAreaZ;
-    [System.NonSerialized] public float minAgentAreaX;
-    [System.NonSerialized] public float maxAgentAreaX;
-    [System.NonSerialized] public float minAgentAreaZ;
-    [System.NonSerialized] public float maxAgentAreaZ;
-
-    [System.NonSerialized] public float nonReward;
-    [System.NonSerialized] public float shootReward;
-    [System.NonSerialized] public float shootWithoutReadyReward;
-    [System.NonSerialized] public float hitReward;
-    [System.NonSerialized] public float winReward;
-    [System.NonSerialized] public float loseReward;
-    [System.NonSerialized] public float killReward;
-    [System.NonSerialized] public float killBonusReward;
-    [System.NonSerialized] public int remainTime;
    [System.NonSerialized] public int finishedState;

-
    private void Start()
    {
-        paramContainer = ParameterContainerObj.GetComponent<parameterContainer>();
-        EnvUICon = EnvironmentUIControlObj.GetComponent<EnviromentUIControl>();
+        paramContainer = ParameterContainerObj.GetComponent<ParameterContainer>();
+        eneContainer = EnemyContainerObj.GetComponent<EnemyContainer>();
+        blockContainer = SceneBlockContainerObj.GetComponent<SceneBlockContainer>();
+        EnvUICon = EnvironmentUIControlObj.GetComponent<EnvironmentUIControl>();
+        targetCon = TargetControllerObj.GetComponent<TargetController>();
        rayScript = GetComponent<RaySensors>();
-        // give default Reward to Reward value will be used.
-        nonReward = paramContainer.nonRewardDefault;
-        shootReward = paramContainer.shootRewardDefault;
-        shootWithoutReadyReward = paramContainer.shootWithoutReadyRewardDefault;
-        hitReward = paramContainer.hitRewardDefault;
-        winReward = paramContainer.winRewardDefault;
-        loseReward = paramContainer.loseRewardDefault;
-        killReward = paramContainer.killRewardDefault;
-        killBonusReward = paramContainer.killBonusRewardDefault;
+        PlayerController = this.transform.GetComponent<CharacterController>();
        // Environment parameters
        lockMouse = paramContainer.lockMouse;
        Damage = paramContainer.Damage;
        fireRate = paramContainer.fireRate;
        enemyNum = paramContainer.enemyNum;
-        timeLimit = paramContainer.timeLimit;
        lockCameraX = paramContainer.lockCameraX;
        lockCameraY = paramContainer.lockCameraY;
        //initialize remainTime
-        remainTime = (int)(timeLimit - Time.time + startTime);
-
-        minEnemyAreaX = edgeLeft.transform.localPosition.x + 1.0f;
-        maxEnemyAreaX = edgeRight.transform.localPosition.x - 1.0f;
-        minEnemyAreaZ = edgeAgent_Enemy.transform.localPosition.z + 1.0f;
-        maxEnemyAreaZ = edgeUp.transform.localPosition.z - 1.0f;
-
-        minAgentAreaX = edgeLeft.transform.localPosition.x + 1.0f;
-        maxAgentAreaX = edgeRight.transform.localPosition.x - 1.0f;
-        minAgentAreaZ = edgeDown.transform.localPosition.z + 1.0f;
-        maxAgentAreaZ = edgeAgent_Enemy.transform.localPosition.z - 1.0f;
+        // this agent's tag
+        myTag = gameObject.tag;
    }

    /* ----------此Update用于debug，Build前删除或注释掉！----------*/
@@ -127,30 +95,6 @@ public class AgentWithGun : Agent
    }*/
    /* ----------此Update用于debug，Build前删除或注释掉！----------*/

-    // --------------初始化---------------
-    // randomInitEnemys随机生成enemy
-    public void randomInitEnemys(int EnemyNum)
-    {
-        for (int i = 0; i < EnemyNum; i++)
-        {
-            float randX = UnityEngine.Random.Range(minEnemyAreaX, maxEnemyAreaX);
-            float randZ = UnityEngine.Random.Range(minEnemyAreaZ, maxEnemyAreaZ);
-            int Y = 1;
-            Instantiate(enemyPrefab, new Vector3(randX, Y, randZ) + EnviromentObj.transform.position, Quaternion.identity, EnemyContainerObj.transform);
-        }
-    }
-
-    // --------------初始化---------------
-    // randomInitAgent随机位置初始化Agent
-    public void randomInitAgent()
-    {
-        float randX = UnityEngine.Random.Range(minAgentAreaX, maxAgentAreaX);
-        float randZ = UnityEngine.Random.Range(minAgentAreaZ, maxAgentAreaZ);
-        int Y = 1;
-        Vector3 initAgentLoc = new Vector3(randX, Y, randZ);
-        thisAgent.localPosition = initAgentLoc;
-    }
-
    // ------------动作处理--------------
    // moveAgent 用于模拟Input.GetAxis移动
    public void moveAgent(int vertical, int horizontal)
@@ -251,7 +195,7 @@ public class AgentWithGun : Agent


        //相机左右旋转时，是以Y轴为中心旋转的，上下旋转时，是以X轴为中心旋转的
-        thisAgent.Rotate(Vector3.up * Mouse_X);
+        transform.Rotate(Vector3.up * Mouse_X);
        //Vector3.up相当于Vector3(0,1,0),CameraRotation.Rotate(Vector3.up * Mouse_X)相当于使CameraRotation对象绕y轴旋转Mouse_X个单位
        //即相机左右旋转时，是以Y轴为中心旋转的，此时Mouse_X控制着值的大小

@@ -265,9 +209,10 @@ public class AgentWithGun : Agent
    }

    // GotKill 获得击杀时用于被呼出
-    public void GotKill()
+    public void killRecord(Vector3 thiskillEnemyPosition)
    {
        enemyKillCount += 1;
+        killEnemyPosition = thiskillEnemyPosition;
    }

    // check gun is ready to shoot
@@ -297,58 +242,26 @@ public class AgentWithGun : Agent
            lastShootTime = Time.time;
            if (Physics.Raycast(ray, out hit, 100))
            {
-                if (hit.collider.tag == "Enemy")
+                if (hit.collider.tag != myTag && hit.collider.tag != "Wall")
                {
                    GameObject gotHitObj = hit.transform.gameObject;//获取受到Ray撞击的对象
-                    gotHitObj.GetComponent<Enemy>().ReactToHit(Damage, thisAgentObj);
+                    gotHitObj.GetComponent<states>().ReactToHit(Damage, gameObject);
                    shoot = 0;
-                    return hitReward;
+                    return paramContainer.hitReward;
                }
            }
            shoot = 0;
-            return shootReward;
+            return paramContainer.shootReward;
        }
        else if (shoot != 0 && gunReadyToggle == false)
        {
            shoot = 0;
-            return shootWithoutReadyReward;
+            return paramContainer.shootWithoutReadyReward;
        }
        else
        {
            shoot = 0;
-            return nonReward;
-        }
-    }
-
-    // destroyEnemy消除EnemyContainer内所有Enemy
-    public void destroyAllEnemys()
-    {
-        foreach (Transform childObj in EnemyContainerObj.transform)
-        {
-            if (childObj.tag == "Enemy")
-            {
-                Destroy(childObj.gameObject);
-            }
-        }
-    }
-
-    // checkFinish 检查是否结束回合返回int值
-    // 1 = success,2 = overtime,0 = notover
-    int checkFinish()
-    {
-        if (EnemyContainerObj.transform.childCount <= 0)
-        {
-            //成功击杀所有Enemy
-            return 1;
-        }
-        else if (Time.time - startTime >= timeLimit)
-        {
-            //超时失败
-            return 2;
-        }
-        else
-        {
-            return 0;
+            return paramContainer.nonReward;
        }
    }

@@ -363,7 +276,7 @@ public class AgentWithGun : Agent
        {
            Vector3 thisEnemyPosition = EnemyObj.transform.localPosition;
            Vector3 thisEnemyScale = EnemyObj.transform.localScale;
-            Vector3 MyselfPosition = thisAgent.localPosition;
+            Vector3 MyselfPosition = transform.localPosition;

            //探测到Agent为自己时的处理
            if (thisEnemyPosition == MyselfPosition)
@@ -389,7 +302,7 @@ public class AgentWithGun : Agent

    // ------------Reward--------------
    // rewardCalculate 计算本动作的Reward
-    public float rewardCalculate()
+    public float rewardCalculate(float sceneReward)
    {
        float epreward = 0f;
        // 击杀reward判断
@@ -397,10 +310,8 @@ public class AgentWithGun : Agent
        {
            for (int i = 0; i < enemyKillCount; i++)
            {
-                epreward += killReward;
-                nonReward += killBonusReward;
-                shootReward += killBonusReward;
-                shootWithoutReadyReward += killBonusReward;
+                // get
+                epreward += targetCon.killReward(killEnemyPosition);
            }
            enemyKillCount = 0;
        }
@@ -409,7 +320,7 @@ public class AgentWithGun : Agent
            enemyKillCount = 0;
        }
        // 射击动作reward判断
-        epreward += ballistic();
+        epreward += ballistic() + sceneReward;
        return epreward;
    }

@@ -423,32 +334,33 @@ public class AgentWithGun : Agent
        {
            Cursor.lockState = CursorLockMode.Locked; // hide and lock the mouse
        }
-        //iniCharts();
        //thisAgentObj.name = thisAgentObj.GetInstanceID().ToString();
-        destroyAllEnemys();
-        startTime = Time.time;// Reset StartTime as now time
-        randomInitAgent();
-        randomInitEnemys(enemyNum);
+        targetCon.rollNewScene();
+        paramContainer.resetTimeBonusReward();
        nowEnemyNum = getEnemyNum(); // Reset Enemy number
        // give default Reward to Reward value will be used.
-        EnvUICon.initChart();
+        if (paramContainer.chartOn)
+        {
+            EnvUICon.initChart();
+        }
    }
-
    // ML-AGENTS处理-------------------------------------------------------------------------------------------ML-AGENTS
    // 观察情报
    public override void CollectObservations(VectorSensor sensor)
    {
-
        //List<float> enemyLDisList = RaySensors.enemyLDisList;// All Enemy Lside Distances
        //List<float> enemyRDisList = RaySensors.enemyRDisList;// All Enemy Rside Distances

        rayScript.updateRayInfo();
-        float[] myObserve = { thisAgent.localPosition.x, thisAgent.localPosition.y, thisAgent.localPosition.z, thisAgent.rotation.w };
+        float[] myObserve = { transform.localPosition.x, transform.localPosition.y, transform.localPosition.z, transform.rotation.w };
        float[] rayTagResult = rayScript.rayTagResult;// 探测用RayTag结果 float[](raySensorNum,1)
        float[] rayDisResult = rayScript.rayDisResult; // 探测用RayDis结果 float[](raySensorNum,1)
+        float[] targetStates = targetCon.getTargetStates(); // targettype, target x,y,z, firebasesAreaDiameter
+
        //float[] focusEnemyObserve = RaySensors.focusEnemyInfo;// 最近的Enemy情报 float[](3,1) MinEnemyIndex,x,z

        //sensor.AddObservation(allEnemyNum); // 敌人数量 int
+        sensor.AddObservation(targetStates);// targettype, target x,y,z, firebasesAreaDiameter
        sensor.AddObservation(myObserve); // 自机位置xyz+朝向 float[](4,1)
        sensor.AddObservation(rayTagResult); // 探测用RayTag结果 float[](raySensorNum,1)
        sensor.AddObservation(rayDisResult); // 探测用RayDis结果 float[](raySensorNum,1)
@@ -470,42 +382,34 @@ public class AgentWithGun : Agent
        float Mouse_X = actionBuffers.ContinuousActions[0];
        if (vertical == 2) vertical = -1;
        if (horizontal == 2) horizontal = -1;
-        remainTime = (int)(timeLimit - Time.time + startTime);

        //应用输入
        shoot = mouseShoot;
        cameraControl(Mouse_X, 0);
        moveAgent(vertical, horizontal);
-        float thisRoundReward = rewardCalculate();

        //判断结束
-        finishedState = checkFinish();
-        if (finishedState == 1)
+        float sceneReward = 0f;
+        (finishedState, sceneReward) = targetCon.checkOverAndRewards();
+        float thisRoundReward = rewardCalculate(sceneReward);
+        if (paramContainer.chartOn)
        {
-            //Win Finished
-            EP += 1;
-            EnvUICon.updateChart(winReward);
-            SetReward(winReward);
-            Debug.Log("reward = " + winReward);
-            EndEpisode();
+            EnvUICon.updateChart(thisRoundReward);
        }
-        else if (finishedState == 2)
+        //Debug.Log("reward = " + thisRoundReward);
+        if (finishedState != (int)TargetController.EndType.Running)
        {
-            //Lose Finished
+            // Win or lose Finished
+            Debug.Log("Finish reward = " + thisRoundReward);
            EP += 1;
-            EnvUICon.updateChart(loseReward);
-            SetReward(loseReward);
-            Debug.Log("reward = " + loseReward);
            EndEpisode();
        }
        else
        {
            // game not over yet
            step += 1;
-            EnvUICon.updateChart(thisRoundReward);
-            SetReward(thisRoundReward);
-            Debug.Log("reward = " + thisRoundReward);
        }
+        SetReward(thisRoundReward);
    }

    // ML-AGENTS处理-------------------------------------------------------------------------------------------ML-AGENTS
@@ -1,40 +1,84 @@
 using System.Collections;
 using System.Collections.Generic;
+using UnityEditor;
 using UnityEngine;

 public class Enemy : MonoBehaviour
 {
-    float EnemyHP = 100;
-    public float EnemyMaxHP = 100;
+    //扇形角度
+    //[SerializeField] private float angle = 80f;
+    //扇形半径
+    //[SerializeField] private float radius = 3.5f;
+    //物体B
+    [SerializeField] private Transform b;
+
+    private bool flag;
+    /// <summary>
+    /// 判断target是否在扇形区域内
+    /// </summary>
+    /// <param name="sectorAngle">扇形角度</param>
+    /// <param name="sectorRadius">扇形半径</param>
+    /// <param name="attacker">攻击者的transform信息</param>
+    /// <param name="target">目标</param>
+    /// <returns>目标target在扇形区域内返回true 否则返回false</returns>

    void Start()
    {
-        EnemyHP = EnemyMaxHP;
    }

    // Update is called once per frame
    void Update()
    {
-        detactDeath();
+        //detactDeath();
+        //flag = IsInRange(angle, radius, transform, b);
    }
-    private void detactDeath()
+
+    public bool IsInRange(float sectorAngle, float sectorRadius, Transform attacker, Transform target)
    {
-        if (EnemyHP <= 0){
-            Destroy(this.gameObject);
-        }
+        //攻击者位置指向目标位置的向量
+        Vector3 direction = target.position - attacker.position;
+        //点乘积结果
+        float dot = Vector3.Dot(direction.normalized, transform.forward);
+        //反余弦计算角度
+        float offsetAngle = Mathf.Acos(dot) * Mathf.Rad2Deg;
+        return offsetAngle < sectorAngle * .5f && direction.magnitude < sectorRadius;
    }
-    public void ReactToHit(float Damage,GameObject damageSource)
+    /*
+    private void OnDrawGizmos()
    {
-        EnemyHP -= Damage;
-        Debug.Log("HP:"+ EnemyHP);
-        if(EnemyHP <= 0)
+        Handles.color = flag ? Color.cyan : Color.red;
+
+        float x = radius * Mathf.Sin(angle / 2f * Mathf.Deg2Rad);
+        float y = Mathf.Sqrt(Mathf.Pow(radius, 2f) - Mathf.Pow(x, 2f));
+        Vector3 a = new Vector3(transform.position.x - x, transform.position.y, transform.position.z + y);
+        Vector3 b = new Vector3(transform.position.x + x, transform.position.y, transform.position.z + y);
+
+        Handles.DrawLine(transform.position, a);
+        Handles.DrawLine(transform.position, b);
+
+        float half = angle / 2;
+        for (int i = 0; i < half; i++)
        {
-            damageSource.GetComponent<AgentWithGun>().GotKill();
-            Destroy(this.gameObject);
+            x = radius * Mathf.Sin((half - i) * Mathf.Deg2Rad);
+            y = Mathf.Sqrt(Mathf.Pow(radius, 2f) - Mathf.Pow(x, 2f));
+            a = new Vector3(transform.position.x - x, transform.position.y, transform.position.z + y);
+            x = radius * Mathf.Sin((half - i - 1) * Mathf.Deg2Rad);
+            y = Mathf.Sqrt(Mathf.Pow(radius, 2f) - Mathf.Pow(x, 2f));
+            b = new Vector3(transform.position.x - x, transform.position.y, transform.position.z + y);
+
+            Handles.DrawLine(a, b);
+        }
+        for (int i = 0; i < half; i++)
+        {
+            x = radius * Mathf.Sin((half - i) * Mathf.Deg2Rad);
+            y = Mathf.Sqrt(Mathf.Pow(radius, 2f) - Mathf.Pow(x, 2f));
+            a = new Vector3(transform.position.x + x, transform.position.y, transform.position.z + y);
+            x = radius * Mathf.Sin((half - i - 1) * Mathf.Deg2Rad);
+            y = Mathf.Sqrt(Mathf.Pow(radius, 2f) - Mathf.Pow(x, 2f));
+            b = new Vector3(transform.position.x + x, transform.position.y, transform.position.z + y);
+
+            Handles.DrawLine(a, b);
        }
    }
-    public float getnowHP()
-    {
-        return EnemyHP;
-    }
+    */
 }
@@ -0,0 +1,67 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public class EnemyContainer : MonoBehaviour
+{
+    public GameObject enemyPrefab;
+    public GameObject EnvironmentObj;
+    public GameObject TargetControllerObj;
+
+    private TargetController targetCon;
+
+    private void Start()
+    {
+        targetCon = TargetControllerObj.GetComponent<TargetController>();
+    }
+
+
+    // initialize enemy by random
+    public void randomInitEnemys(int EnemyNum)
+    {
+        for (int i = 0; i < EnemyNum; i++)
+        {
+            float randX = UnityEngine.Random.Range(targetCon.minEnemyAreaX, targetCon.maxEnemyAreaX);
+            float randZ = UnityEngine.Random.Range(targetCon.minEnemyAreaZ, targetCon.maxEnemyAreaZ);
+            int enemyY = 1;
+            initEnemyAtHere(new Vector3(randX, enemyY, randZ));
+        }
+    }
+
+    // initialize enemy by random but not in block area
+    public void randomInitEnemysExcept(int enemyNum,Vector3 blockPosition,float sceneSize)
+    {
+        float randX = 0f;
+        float randZ = 0f;
+        for (int i = 0; i < enemyNum; i++)
+        {
+            randX = UnityEngine.Random.Range(targetCon.minEnemyAreaX, targetCon.maxEnemyAreaX);
+            randZ = UnityEngine.Random.Range(targetCon.minEnemyAreaZ, targetCon.maxEnemyAreaZ);
+            while (Vector3.Distance(blockPosition, new Vector3(randX,0f,randZ)) < sceneSize/2)
+            {
+                // while in scene area then respawn
+                Debug.Log("spawn enemy in area, re:roll");
+                randX = UnityEngine.Random.Range(targetCon.minEnemyAreaX, targetCon.maxEnemyAreaX);
+                randZ = UnityEngine.Random.Range(targetCon.minEnemyAreaZ, targetCon.maxEnemyAreaZ);
+            }
+
+            int enemyY = 1;
+            initEnemyAtHere(new Vector3(randX, enemyY, randZ));
+        }
+    }
+
+    // initialize enemy to thisPosition
+    public void initEnemyAtHere(Vector3 thisPosition)
+    {
+        Instantiate(enemyPrefab, thisPosition + EnvironmentObj.transform.position, Quaternion.identity, this.transform);
+    }
+
+    // destroyEnemy delete enemyContainer's all enemy
+    public void destroyAllEnemys()
+    {
+        foreach (Transform childObj in this.transform)
+        {
+            childObj.GetComponent<states>().destroyMe();
+        }
+    }
+}
@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: b293d3b541b20b7448f3ca4d09a9c38c
+guid: 0350ec966da42b44699fee7309a89ac7
 MonoImporter:
  externalObjects: {}
  serializedVersion: 2
@@ -0,0 +1,114 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using TMPro;
+using UnityEngine;
+using UnityEngine.UI;
+using XCharts.Runtime;
+
+public class EnvironmentUIControl : MonoBehaviour
+{
+    public GameObject TargetControllerObj;
+    public GameObject ParameterContainerObj;
+    public GameObject GroundCanvasObj;
+    public GameObject chartObj;
+    public TextMeshProUGUI remainTimeText;
+    public TextMeshProUGUI winLoseText;
+    public float resultTimeout = 1f;
+    public GameObject gaugeImgObj;
+
+    private LineChart realTimeRewardChart = null;
+    private TargetController targetController;
+    private ParameterContainer paramContainer;
+    private Image gaugeImg;
+    private float overTime = 0f;
+    private int step = 0;
+    private bool resultActive = false;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        targetController = TargetControllerObj.GetComponent<TargetController>();
+        paramContainer = ParameterContainerObj.GetComponent<ParameterContainer>();
+        gaugeImg = gaugeImgObj.GetComponent<Image>();
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        int remainTime = Convert.ToInt32(targetController.startTime + paramContainer.timeLimit - Time.time);
+        remainTimeText.text = "RemainTime:" + remainTime.ToString();
+        if (resultActive && Time.time - overTime >= resultTimeout)
+        {
+            // while result is active and show time over timeOut
+            // clear the result
+            Debug.Log("clear result");
+            winLoseText.text = "";
+            winLoseText.color = Color.white;
+            resultActive = false;
+        }
+    }
+    public void updateChart(float reward)
+    {
+        step += 1;
+        realTimeRewardChart.AddXAxisData(Convert.ToString(step));
+        realTimeRewardChart.AddData(0, reward);
+    }
+
+    public void initChart()
+    {
+        if (paramContainer.chartOn && realTimeRewardChart == null)
+        {
+            Vector3 chartPos = new Vector3(-210f, 90f, 0f) * GroundCanvasObj.transform.localScale.x;
+            realTimeRewardChart = chartObj.AddComponent<LineChart>();
+            realTimeRewardChart.Init();
+        }
+        realTimeRewardChart.RemoveData();
+        realTimeRewardChart.AddSerie<Line>("Rewards");
+    }
+
+    // show result in UI
+    public void showResult(int resultState)
+    {
+        switch (resultState)
+        {
+            case (int)TargetController.EndType.Win:
+                //Win
+                Debug.Log("win");
+                winLoseText.text = "Win";
+                winLoseText.color = Color.green;
+                overTime = Time.time;
+                resultActive = true;
+                break;
+            case (int)TargetController.EndType.Lose:
+                //lose
+                Debug.Log("lose");
+                winLoseText.text = "Lose";
+                winLoseText.color = Color.red;
+                overTime = Time.time;
+                resultActive = true;
+                break;
+            default:
+                break;
+        }
+    }
+
+    // update firebases target state gauge
+    public void updateTargetGauge(float firebasesBelong, float belongMaxPoint)
+    {
+        if (firebasesBelong >= 0)
+        {
+            gaugeImgObj.transform.localPosition = new Vector3(25f, 0, 0);
+            gaugeImg.color = Color.blue;
+            gaugeImg.fillOrigin = 0;
+            gaugeImg.fillAmount = firebasesBelong / belongMaxPoint;
+        }
+        else
+        {
+            gaugeImgObj.transform.localPosition = new Vector3(-25f, 0, 0);
+            gaugeImg.color = Color.red;
+            gaugeImg.fillOrigin = 1;
+            gaugeImg.fillAmount = -firebasesBelong / belongMaxPoint;
+        }
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: e3ddb0247fbea1b4799dd33b60ab4f12
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -3,22 +3,22 @@ using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.UI;

-public class EnemyHPBar : MonoBehaviour
+public class HPBar : MonoBehaviour
 {
-    GameObject EnemyOBJ;
+    GameObject thisObj;
    GameObject BGOBJ;
    GameObject gaugeImgOBJ;
-    
+

    void Start()
    {
-        EnemyOBJ = transform.parent.gameObject;
+        thisObj = transform.parent.gameObject;
        BGOBJ = transform.GetChild(0).gameObject;
        gaugeImgOBJ = BGOBJ.transform.GetChild(0).gameObject;
        Vector3 v = Camera.main.transform.position - transform.position;
        v.x = v.z = 0.0f;
        transform.LookAt(Camera.main.transform.position - v);
-        transform.Rotate(0,180,0);
+        transform.Rotate(0, 180, 0);
    }

    void Update()
@@ -26,10 +26,10 @@ public class EnemyHPBar : MonoBehaviour
        Vector3 v = Camera.main.transform.position - transform.position;
        v.x = v.z = 0.0f;
        transform.LookAt(Camera.main.transform.position - v);
-        transform.Rotate(0,180,0);
+        transform.Rotate(0, 180, 0);

-        float maxHP = EnemyOBJ.GetComponent<Enemy>().EnemyMaxHP;
-        float nowHP = EnemyOBJ.GetComponent<Enemy>().getnowHP();
+        float maxHP = thisObj.GetComponent<states>().MaxHP;
+        float nowHP = thisObj.GetComponent<states>().getnowHP();
        gaugeImgOBJ.GetComponent<Image>().fillAmount = nowHP / maxHP;
    }
 }
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 925561829acf6c94097de75bf186b561
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -0,0 +1,130 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public class ParameterContainer : MonoBehaviour
+{
+    public GameObject targetConObj;
+    public GameObject blockConObj;
+    public GameObject agentObj;
+    private TargetController targetCon;
+    private SceneBlockContainer blockCont;
+    private float agentDistance;
+    private int agentInArea;
+
+
+    [Header("Env")]
+    public bool lockMouse = false;
+    public float Damage = 50; // damage to enemy
+    public float fireRate = 0.5f;
+    public int enemyNum = 3;
+    public int timeLimit = 30;
+    public bool lockCameraX = false;
+    public bool lockCameraY = true;
+    public bool chartOn = false;
+
+
+    [Header("Dynamic Defaut Rewards")]
+    [Tooltip("Hit Enemy reward")]
+    public float hitRewardDefault = 30.0f;
+    [Tooltip("Episode Win reward")]
+    public float winRewardDefault = 50.0f;
+    [Tooltip("Enemy down reward")]
+    public float killRewardDefault = 40.0f;
+    [Tooltip("Enemy down in area Reward")]
+    public float killInAreaEnemyRewardDefault = 80.0f;
+    [Tooltip("stay in firebasesArea reward")]
+    public float inAreaRewardDefault = 1.0f;
+    [Tooltip("free left time bonus reward. ALLR + leftTime * r")]
+    public float freeTimeBonusPerSec = 1.0f;
+    [Tooltip("target left time bonus reward. ALLR + leftTime * r")]
+    public float targetTimeBonusPerSec = 3.0f;
+    [Tooltip("in area left time bonus reward. ALLR + leftTime * r")]
+    public float areaTimeBonusPerSec = 1.0f;
+    [Tooltip("distance reward reward = r*(1-(nowDis/startDis))")]
+    public float distanceReward = 1.0f;
+
+    [Header("Dynamic Rewards")]
+    [Tooltip("Hit Enemy reward")]
+    public float hitReward = 30.0f;
+    [Tooltip("Enemy down reward")]
+    public float killReward = 40.0f;
+    [Tooltip("Episode Win reward")]
+    public float winReward = 50.0f;
+    [Tooltip("Enemy down in area Reward")]
+    public float killInAreaEnemyReward = 80.0f;
+    [Tooltip("stay in firebasesArea reward")]
+    public float inAreaReward = 1.0f;
+
+
+
+    [Header("Static Rewards")]
+    [Tooltip("Nothing happened reward")]
+    public float nonReward = -0.05f;
+    [Tooltip("Episode Lose reward")]
+    public float loseReward = -0.05f;
+    [Tooltip("Agent Do shoot action reward")]
+    public float shootReward = -0.1f;
+    [Tooltip("Agent Do shoot action but gun is not read")]
+    public float shootWithoutReadyReward = -0.15f;
+    [Tooltip("Kill bonus reward stack to nothing happend reward")]
+    public float killBonusReward = 0.0f;
+    [Header("Penalty Rewards")]
+    [Tooltip("Speed Penalty Reward")]
+    public float speedPanalty = 0f;
+    [Tooltip("view Panalty Reward")]
+    public float viewPanalty = 0f;
+
+    private float targetTimeBonus = 0f;
+    private float areaTimeBonus = 0f;
+    private float freeTimeBonus = 0f;
+    private float targetInAreaTime = 0f;
+    private float lastFrameTime = 0f;
+
+    private void Start()
+    {
+        targetCon = targetConObj.GetComponent<TargetController>();
+        blockCont = blockConObj.GetComponent<SceneBlockContainer>();
+        areaTimeBonus = areaTimeBonusPerSec * timeLimit;
+        freeTimeBonus = freeTimeBonusPerSec * timeLimit;
+        targetTimeBonus = targetTimeBonusPerSec * timeLimit;
+    }
+
+    private void Update()
+    {
+        if (targetCon.targetTypeInt != (int)TargetController.Targets.Free)
+        {
+            (agentDistance, agentInArea) = blockCont.getAgentTargetDistanceAndInside(agentObj.transform.position);
+            // attack goto or defence target
+            if (agentInArea == 1)
+            {
+                // agent out of area
+                targetInAreaTime += Time.time - lastFrameTime;
+            }
+            areaTimeBonus = areaTimeBonusPerSec * (targetCon.leftTime + targetInAreaTime);
+            freeTimeBonus = freeTimeBonusPerSec * targetCon.leftTime;
+            targetTimeBonus = targetTimeBonusPerSec * targetCon.leftTime;
+            lastFrameTime = Time.time;
+        }
+        else
+        {
+            // free target
+            areaTimeBonus = areaTimeBonusPerSec * targetCon.leftTime;
+            freeTimeBonus = freeTimeBonusPerSec * targetCon.leftTime;
+            targetTimeBonus = targetTimeBonusPerSec * targetCon.leftTime;
+        }
+
+        hitReward = hitRewardDefault + freeTimeBonus;
+        killReward = killRewardDefault + freeTimeBonus;
+        winReward = winRewardDefault + targetTimeBonus;
+        killInAreaEnemyReward = killInAreaEnemyRewardDefault + targetTimeBonus;
+        inAreaReward = inAreaRewardDefault + areaTimeBonus;
+    }
+
+    public void resetTimeBonusReward()
+    {
+        areaTimeBonus = areaTimeBonusPerSec * timeLimit;
+        freeTimeBonus = freeTimeBonusPerSec * timeLimit;
+        targetInAreaTime = 0f;
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: cab5ee46e8af1134b840e91077e7592b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -31,6 +31,7 @@ public class RaySensors : MonoBehaviour
    public float[] rayDisResult;

    [System.NonSerialized] public int totalRayNum;
+    private string myTag = "";
    GameObject[] linesOBJ;
    GameObject[] rayInfoOBJ;
    LineRenderer[] lineRenderers;
@@ -39,7 +40,7 @@ public class RaySensors : MonoBehaviour

    private void Start()
    {
-        
+        myTag = gameObject.tag;
        totalRayNum = halfOuterRayNum * 2 + focusRayNum;
        rayTagResult = new float[totalRayNum];
        rayDisResult = new float[totalRayNum];
@@ -63,15 +64,17 @@ public class RaySensors : MonoBehaviour
        }
    }

-    static int tagToInt(string tag)
+    public int tagToInt(string tag)
    {
        switch (tag)
        {
            case "Wall":
                return 1;
-            case "Enemy":
-                return 2;
            default:
+                if(tag != myTag)
+                {
+                    return 2;
+                }
                return 0;
        }
    }
@@ -0,0 +1,197 @@
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+
+public class SceneBlock : MonoBehaviour
+{
+    public GameObject FirebasesAreaObj;
+    public GameObject EnemyContainerObj;
+    public float group1InareaNum = 0;
+    public float group2InareaNum = 0;
+    public float belongRatio = 0;
+    public float blockSize = 10f;
+    public Vector3 firebasesAreaPosition;
+    public float firebasesAreaScale;
+    public float firebasesAreaDiameter;
+
+    public enum SceneType { Go, Attack, Defence }
+    public SceneType sceneType;
+    public bool isDestroyed = false;
+
+    // firebase state
+    public string group1Tag = "Player";
+    public string group2Tag = "Enemy";
+    public float belongMaxPoint = 10f;
+    public float firebasesBelong; // -10 mean's belon to group2Tag 10 = belon to group1Tag
+    public float addPointInterval = 0.1f; // add point every addPointInterval scecond
+    public float stayTimeNeeded = 5f; // howmany time to stay at area to get this area 0->10
+
+    private float addPointEachInterval; // add point in each interval
+    private float intervalStart;
+    private GameObject EnvironmentObj;
+    public GameObject[] group1Objs;
+    public GameObject[] group2Objs;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        firebasesAreaPosition = transform.position + FirebasesAreaObj.transform.position;
+        firebasesAreaScale = FirebasesAreaObj.transform.localScale.x;
+        firebasesAreaDiameter = firebasesAreaScale * blockSize;
+        firebasesBelong = -belongMaxPoint;
+        addPointEachInterval = belongMaxPoint / (stayTimeNeeded / addPointInterval);
+        intervalStart = Time.time;
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        if (Time.time - intervalStart >= addPointInterval)
+        {
+            group1InareaNum = getInAreaNumber(group1Tag);
+            group2InareaNum = getInAreaNumber(group2Tag);
+            belongRatio = group1InareaNum - group2InareaNum;
+            if (belongRatio > 0)
+            {
+                // group 1 have the advantage!
+                firebasesBelong += addPointEachInterval;
+                if (firebasesBelong >= belongMaxPoint)
+                {
+                    firebasesBelong = belongMaxPoint;
+                }
+            }
+            else if (belongRatio < 0)
+            {
+                // group 2 have the advantage!
+                firebasesBelong -= addPointEachInterval;
+                if (firebasesBelong <= -belongMaxPoint)
+                {
+                    firebasesBelong = -belongMaxPoint;
+                }
+            }
+            intervalStart = Time.time;
+        }
+    }
+
+    //Initialize thi scene block should be excuted after enemy created
+    public void initBlock(GameObject envObj)
+    {
+        //Buffer all Player or enemy obj int this environment to list
+        EnvironmentObj = envObj;
+        GameObject[] allGroup1Objs = GameObject.FindGameObjectsWithTag(group1Tag);
+        GameObject[] allGroup2Objs = GameObject.FindGameObjectsWithTag(group2Tag);
+        List<GameObject> group1ObjsList = new List<GameObject>();
+        List<GameObject> group2ObjsList = new List<GameObject>();
+        foreach (GameObject obj in allGroup1Objs)
+        {
+            if (obj.transform.root.gameObject == envObj && !obj.GetComponent<states>().isDead)
+            {
+                group1ObjsList.Add(obj);
+            }
+        }
+        foreach (GameObject obj in allGroup2Objs)
+        {
+            if(obj.transform.root.gameObject == envObj && !obj.GetComponent<states>().isDead)
+            {
+                group2ObjsList.Add(obj);
+            }
+        }
+        group1Objs = group1ObjsList.ToArray();
+        group2Objs = group2ObjsList.ToArray();
+    }
+
+    //check game over 0=notover 1=win
+    public int checkOver()
+    {
+        return 0;
+    }
+
+    // get in area player number by tag
+    public float getInAreaNumber(string thisTag)
+    {
+        float inAreaNum = 0;
+        float dist = 0f;
+        float isInarea = 0f;
+        int index = 0;
+        if (thisTag == group1Tag)
+        {
+            foreach(GameObject obj in group1Objs)
+            {
+                // if object is dead then delete it from list
+                if (obj == null)
+                {
+                    // delete null object from list
+                    List<GameObject> group1ObjsList = new List<GameObject>(group1Objs);
+                    group1ObjsList.RemoveAt(index);
+                    group1Objs = group1ObjsList.ToArray();
+                }
+                else
+                {
+                    (dist, isInarea) = getDist_inArea(obj.transform.position);
+                    if (isInarea != 0f)
+                    {
+                        inAreaNum += 1;
+                    }
+                }
+                index++;
+            }
+        }else if(thisTag == group2Tag)
+        {
+            foreach(GameObject obj in group2Objs)
+            {
+                // if object is dead then delete it from list
+                if (obj == null)
+                {
+                    // delete null object from list
+                    List<GameObject> group2ObjsList = new List<GameObject>(group2Objs);
+                    group2ObjsList.RemoveAt(index);
+                    group2Objs = group2ObjsList.ToArray();
+                }
+                else
+                {
+                    (dist, isInarea) = getDist_inArea(obj.transform.position);
+                    if (isInarea != 0f)
+                    {
+                        inAreaNum += 1;
+                    }
+                }
+                index++;
+            }
+        }
+        else
+        {
+            Debug.LogError("SceneBlock.getinAreaNumber:TagError");
+        }
+        return inAreaNum;
+    }
+
+    // get this position and target's distance and is in firebase area
+    public (float, int) getDist_inArea(Vector3 thisPosition)
+    {
+        thisPosition.y = FirebasesAreaObj.transform.position.y;
+        float dist = Vector3.Distance(thisPosition, FirebasesAreaObj.transform.position) - (firebasesAreaDiameter/2);
+        int isinarea = 0;
+        if (dist <= 0)
+        {
+            dist = 0f;
+            isinarea = 1;
+        }
+        else
+        {
+            isinarea = 0;
+        }
+        return (dist, isinarea);
+    }
+
+    //destroy this block
+    public void destroyMe()
+    {
+        Destroy(this.gameObject);
+        foreach (Transform childObj in EnemyContainerObj.transform)
+        {
+            childObj.GetComponent<states>().destroyMe();
+        }
+        isDestroyed = true;
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 8e1ae09061637cf4ead72321c221f783
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -0,0 +1,77 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public class SceneBlockContainer : MonoBehaviour
+{
+    public float sceneSize = 10f;
+    public GameObject EnvironmentObj;
+    public GameObject[] attackBlockPrefabs = new GameObject[1];
+    public GameObject[] goBlockPrefabs = new GameObject[1];
+    public GameObject[] defencePrefabs = new GameObject[1];
+
+    private GameObject thisBlockObj;
+    public SceneBlock thisBlock;
+
+    private void Start()
+    {
+    }
+    // create block random
+    public void createNewBlock(int targetType, int blockType, Vector3 blockPosition,string tag1,string tag2)
+    {
+        // check if thisBlock is deleted
+        if (thisBlockObj != null)
+        {
+            // delete thisBlock
+            Debug.LogWarning("Block not clear!");
+            destroyBlock();
+        }
+        // choose target type
+        switch (targetType)
+        {
+            case (int)TargetController.Targets.Go:
+                // goto
+                thisBlockObj = Instantiate(goBlockPrefabs[blockType], blockPosition + EnvironmentObj.transform.position, Quaternion.identity, transform);
+                thisBlock = thisBlockObj.GetComponent<SceneBlock>();
+                thisBlock.group1Tag = tag1;
+                thisBlock.group2Tag = tag2;
+                sceneSize = thisBlock.blockSize;
+                break;
+            case (int)TargetController.Targets.Attack:
+                // attack
+                thisBlockObj = Instantiate(attackBlockPrefabs[blockType], blockPosition+ EnvironmentObj.transform.position, Quaternion.identity, transform);
+                thisBlock = thisBlockObj.GetComponent<SceneBlock>();
+                thisBlock.group1Tag = tag1;
+                thisBlock.group2Tag = tag2;
+                sceneSize = thisBlock.blockSize;
+                break;
+            case (int)TargetController.Targets.Defence:
+                // defence
+                thisBlockObj = Instantiate(defencePrefabs[blockType], blockPosition + EnvironmentObj.transform.position, Quaternion.identity, transform);
+                thisBlock = thisBlockObj.GetComponent<SceneBlock>();
+                thisBlock.group1Tag = tag1;
+                thisBlock.group2Tag = tag2;
+                sceneSize = thisBlock.blockSize;
+                break;
+        }
+    }
+
+    // delete thisBlock
+    public void destroyBlock()
+    {
+        if (thisBlock != null)
+        {
+            thisBlock.destroyMe();
+        }
+        thisBlockObj = null;
+        thisBlock = null;
+    }
+
+    public (float, int) getAgentTargetDistanceAndInside(Vector3 agentPosition)
+    {
+        return thisBlock.getDist_inArea(agentPosition);
+    }
+
+
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 94687c2b580b85d4a8ced43c8cfc7bf2
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -0,0 +1,379 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Random = UnityEngine.Random;
+
+public class TargetController : MonoBehaviour
+{
+    public GameObject EnvironmentObj;
+    public GameObject AgentObj;
+    public GameObject SceneBlockContainerObj;
+    public GameObject EnemyContainerObj;
+    public GameObject ParameterContainerObj;
+    public GameObject EnvironmentUIObj;
+    public GameObject WorldUIObj;
+    // area
+    public GameObject edgeUp;
+    public GameObject edgeDown;
+    public GameObject edgeLeft;
+    public GameObject edgeRight;
+    public GameObject edgeAgent_Enemy;
+
+    //group
+    public string group1Tag = "Player";
+    public string group2Tag = "Enemy";
+
+    public float minEnemyAreaX;
+    [System.NonSerialized] public float maxEnemyAreaX;
+    [System.NonSerialized] public float minEnemyAreaZ;
+    [System.NonSerialized] public float maxEnemyAreaZ;
+    [System.NonSerialized] public float minAgentAreaX;
+    [System.NonSerialized] public float maxAgentAreaX;
+    [System.NonSerialized] public float minAgentAreaZ;
+    [System.NonSerialized] public float maxAgentAreaZ;
+    [System.NonSerialized] public float startTime = 0f;
+    [System.NonSerialized] public float leftTime = 0f;
+
+    [SerializeField, Range(0f, 1f)] public float attackProb = 0.2f;
+    [SerializeField, Range(0f, 1f)] public float gotoProb = 0.2f;
+    [SerializeField, Range(0f, 1f)] public float defenceProb = 0.2f;
+
+    [System.NonSerialized] public int targetTypeInt;
+    public enum Targets { Free, Go, Attack, Defence, Num };// Num is use for get total target bumber
+    public enum EndType { Win, Lose, Running, Num };
+    [System.NonSerialized] public int targetNum = 0;
+    private Dictionary<int, float[]> oneHotRarget = new Dictionary<int, float[]>();
+
+    private float freeProb;
+    private float sceneSize;
+    private float lastDistance;
+    private Vector3 targetPosition;
+    private Vector3 targetLocalPosition;
+    private bool firstRewardFlag = true;
+    private SceneBlockContainer blockCont;
+    private EnemyContainer enemyCont;
+    private EnvironmentUIControl envUICon;
+    private ParameterContainer paramCon;
+    private CharacterController agentCharaCon;
+    private WorldUIController worldUICon;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        minEnemyAreaX = edgeLeft.transform.localPosition.x + 1.0f;
+        maxEnemyAreaX = edgeRight.transform.localPosition.x - 1.0f;
+        minEnemyAreaZ = edgeAgent_Enemy.transform.localPosition.z + 1.0f;
+        maxEnemyAreaZ = edgeUp.transform.localPosition.z - 1.0f;
+
+        minAgentAreaX = edgeLeft.transform.localPosition.x + 1.0f;
+        maxAgentAreaX = edgeRight.transform.localPosition.x - 1.0f;
+        minAgentAreaZ = edgeDown.transform.localPosition.z + 1.0f;
+        maxAgentAreaZ = edgeAgent_Enemy.transform.localPosition.z - 1.0f;
+
+        blockCont = SceneBlockContainerObj.GetComponent<SceneBlockContainer>();
+        envUICon = EnvironmentUIObj.GetComponent<EnvironmentUIControl>();
+        enemyCont = EnemyContainerObj.GetComponent<EnemyContainer>();
+        agentCharaCon = AgentObj.GetComponent<CharacterController>();
+        paramCon = ParameterContainerObj.GetComponent<ParameterContainer>();
+        worldUICon = WorldUIObj.GetComponent<WorldUIController>();
+        freeProb = 1 - attackProb - gotoProb - defenceProb;
+        targetNum = (int)Targets.Num;
+        if (freeProb < 0)
+        {
+            Debug.LogError("target percentage wrong");
+        }
+
+        // initialize a simple fake onehot encoder.
+        for (int i = 0; i < targetNum; i++)
+        {
+            float[] onehotList = new float[targetNum];
+            for (int j = 0; j < targetNum; j++)
+            {
+                onehotList[j] = 0;
+            }
+            onehotList[i] = 1;
+            oneHotRarget.Add(i, onehotList);
+        }
+    }
+
+    private void Update()
+    {
+        leftTime = paramCon.timeLimit - Time.time + startTime;
+    }
+
+    public void rollNewScene()
+    {
+        startTime = Time.time;// Reset StartTime as now time
+        float randTargetType = UnityEngine.Random.Range(0f, 1f);
+        if (randTargetType <= gotoProb)
+        {
+            // goto target spawn
+            Debug.Log("GOTO THIS TARGET!");
+            targetTypeInt = (int)Targets.Go;
+            int randBlockType = Random.Range(0, blockCont.goBlockPrefabs.Length);
+            // get choosed scene size 
+            sceneSize = blockCont.goBlockPrefabs[randBlockType].GetComponent<SceneBlock>().blockSize;
+            float randX = UnityEngine.Random.Range(minEnemyAreaX + sceneSize + 1f, maxEnemyAreaX - sceneSize - 1f);
+            float randZ = UnityEngine.Random.Range(minEnemyAreaZ + sceneSize + 1f, maxEnemyAreaZ - sceneSize - 1f);
+            targetPosition = new Vector3(randX, 0f, randZ);
+            // Init Agent position
+            moveAgentToSpwanArea();
+            // init scene block
+            blockCont.destroyBlock();
+            blockCont.createNewBlock(targetTypeInt, randBlockType, targetPosition, group1Tag, group2Tag);
+            enemyCont.destroyAllEnemys();
+            enemyCont.randomInitEnemysExcept(paramCon.enemyNum, targetPosition, sceneSize);
+            blockCont.thisBlock.initBlock(EnvironmentObj);
+            // set startDistance
+            firstRewardFlag = true;
+        }
+        else if (randTargetType > gotoProb && randTargetType <= gotoProb + attackProb)
+        {
+            // attack target spawn
+            Debug.Log("ATTACK!");
+            targetTypeInt = (int)Targets.Attack;
+            int randBlockType = Random.Range(0, blockCont.attackBlockPrefabs.Length);
+            // get choosed scene size 
+            sceneSize = blockCont.attackBlockPrefabs[randBlockType].GetComponent<SceneBlock>().blockSize;
+            float randX = UnityEngine.Random.Range(minEnemyAreaX + sceneSize + 1f, maxEnemyAreaX - sceneSize - 1f);
+            float randZ = UnityEngine.Random.Range(minEnemyAreaZ + sceneSize + 1f, maxEnemyAreaZ - sceneSize - 1f);
+            targetPosition = new Vector3(randX, 0f, randZ);
+            // Init Agent position
+            moveAgentToSpwanArea();
+            // init scene block
+            blockCont.destroyBlock();
+            blockCont.createNewBlock(targetTypeInt, randBlockType, targetPosition, group1Tag, group2Tag);
+            enemyCont.destroyAllEnemys();
+            enemyCont.randomInitEnemysExcept(paramCon.enemyNum, targetPosition, sceneSize);
+            blockCont.thisBlock.initBlock(EnvironmentObj);
+            // set startDistance
+            firstRewardFlag = true;
+        }
+        else if (randTargetType > gotoProb + attackProb && randTargetType <= gotoProb + attackProb + defenceProb)
+        {
+            // defence target spawn
+            Debug.Log("DEFENCE!");
+            targetTypeInt = (int)Targets.Defence;
+            int randBlockType = Random.Range(0, blockCont.attackBlockPrefabs.Length);
+            // get choosed scene size 
+            sceneSize = blockCont.defencePrefabs[randBlockType].GetComponent<SceneBlock>().blockSize;
+            float randX = UnityEngine.Random.Range(minEnemyAreaX + sceneSize + 1f, maxEnemyAreaX - sceneSize - 1f);
+            float randZ = UnityEngine.Random.Range(minEnemyAreaZ + sceneSize + 1f, maxEnemyAreaZ - sceneSize - 1f);
+            targetPosition = new Vector3(randX, 0f, randZ);
+            // Init Agent position
+            moveAgentTo(targetPosition);
+            // init scene block
+            blockCont.destroyBlock();
+            blockCont.createNewBlock(targetTypeInt, randBlockType, targetPosition, group1Tag, group2Tag);
+            enemyCont.destroyAllEnemys();
+            enemyCont.randomInitEnemysExcept(paramCon.enemyNum, targetPosition, sceneSize);
+            blockCont.thisBlock.initBlock(EnvironmentObj);
+            // set startDistance
+            firstRewardFlag = true;
+        }
+        else
+        {
+            Debug.Log("Free");
+            targetTypeInt = (int)Targets.Free;
+            enemyCont.destroyAllEnemys();
+            enemyCont.randomInitEnemys(paramCon.enemyNum);
+            moveAgentToSpwanArea();
+            blockCont.destroyBlock();
+        }
+    }
+
+    // get target observation states
+    public float[] getTargetStates()
+    {
+        // targettype, x,y,z, firebasesAreaDiameter
+        List<float> targetState = new List<float>();
+        if (targetTypeInt == (int)Targets.Free)
+        {
+            targetState.AddRange(oneHotRarget[targetTypeInt]);
+            targetState.AddRange(new float[5] { 0f, 0f, 0f, 0f, 0f });
+        }
+        else
+        {
+            targetState.AddRange(oneHotRarget[targetTypeInt]);
+            targetState.AddRange(new float[5] { targetPosition.x, targetPosition.y, targetPosition.z, blockCont.thisBlock.firebasesAreaDiameter, blockCont.thisBlock.belongRatio });
+        }
+        return targetState.ToArray();
+    }
+
+    // move Agent into Agent Spawn Area
+    public void moveAgentToSpwanArea()
+    {
+        float randX = UnityEngine.Random.Range(minAgentAreaX, maxAgentAreaX);
+        float randZ = UnityEngine.Random.Range(minAgentAreaZ, maxAgentAreaZ);
+        int Y = 1;
+        Vector3 initAgentLoc = new Vector3(randX, Y, randZ);
+        moveAgentTo(initAgentLoc);
+    }
+
+    // move Agent to this position
+    public void moveAgentTo(Vector3 thisPosition)
+    {
+        // while using transform.localPosition to move character
+        // u should turn off character Controller or it won't work
+        agentCharaCon.enabled = false;
+        AgentObj.transform.localPosition = thisPosition;
+        agentCharaCon.enabled = true;
+    }
+
+    // caulculate sceneReward if close to target then get great reward
+    public float getSceneReward(float nowDistance)
+    {
+        if (firstRewardFlag)
+        {
+            (lastDistance, _) = blockCont.getAgentTargetDistanceAndInside(AgentObj.transform.position);
+            firstRewardFlag = false;
+        }
+        float thisSceneReward = 0f;
+        thisSceneReward = paramCon.distanceReward * (lastDistance - nowDistance);
+        lastDistance = nowDistance;
+        return thisSceneReward;
+    }
+
+    // check over and get rewards
+    // 1 = success,2 = overtime,0 = notover
+    public (int, float) checkOverAndRewards()
+    {
+        int endTypeInt = 0;
+        float thisReward = 0;
+        int inArea = 0;
+        float nowDistance = 0f;
+        switch (targetTypeInt)
+        {
+            case (int)Targets.Go:
+                // goto
+                (nowDistance, inArea) = blockCont.getAgentTargetDistanceAndInside(AgentObj.transform.position);
+                envUICon.updateTargetGauge(blockCont.thisBlock.firebasesBelong, blockCont.thisBlock.belongMaxPoint);
+                if (blockCont.thisBlock.firebasesBelong >= blockCont.thisBlock.belongMaxPoint)
+                {
+                    // win
+                    // let the area belongs to me
+                    thisReward = paramCon.winReward;
+                    //thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Win;
+                }
+                else if (leftTime <= 0)
+                {
+                    // time out lose
+                    thisReward = paramCon.loseReward;
+                    //thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Lose;
+                }
+                else
+                {
+                    // keep on keeping on!
+
+                    thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Running;
+                }
+                break;
+            case (int)Targets.Attack:
+                // attack
+                (nowDistance, inArea) = blockCont.getAgentTargetDistanceAndInside(AgentObj.transform.position);
+                envUICon.updateTargetGauge(blockCont.thisBlock.firebasesBelong, blockCont.thisBlock.belongMaxPoint);
+                if (blockCont.thisBlock.firebasesBelong >= blockCont.thisBlock.belongMaxPoint && blockCont.thisBlock.getInAreaNumber(group2Tag) <= 0)
+                {
+                    // win
+                    // let the area belongs to me and kill every enmy in this area.
+                    thisReward = paramCon.winReward;
+                    //thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Win;
+                }
+                else if (leftTime <= 0)
+                {
+                    // time out lose
+                    thisReward = paramCon.loseReward;
+                    //thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Lose;
+                }
+                else
+                {
+                    // keep on keeping on!
+                    thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Running;
+                }
+                break;
+            case (int)Targets.Defence:
+                //defence
+                (nowDistance, inArea) = blockCont.getAgentTargetDistanceAndInside(AgentObj.transform.position);
+                envUICon.updateTargetGauge(blockCont.thisBlock.firebasesBelong, blockCont.thisBlock.belongMaxPoint);
+                if (leftTime <= 0 && blockCont.thisBlock.firebasesBelong >= 0f)
+                {
+                    // win
+                    // time over and the area still mine
+                    thisReward = paramCon.winReward;
+                    //thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Win;
+                }
+                else if (blockCont.thisBlock.firebasesBelong <= blockCont.thisBlock.belongMaxPoint)
+                {
+                    // lost area lose
+                    thisReward = paramCon.loseReward;
+                    //thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Lose;
+                }
+                else
+                {
+                    // keep on keeping on!
+                    thisReward = (paramCon.inAreaReward * inArea) + getSceneReward(nowDistance);
+                    endTypeInt = (int)EndType.Running;
+                }
+                break;
+            default:
+                //free kill
+                if (EnemyContainerObj.transform.childCount <= 0)
+                {
+                    // win
+                    //thisReward = paramCon.winReward + (paramCon.timeBonusPerSecReward * leftTime);
+                    thisReward = 0f;
+                    endTypeInt = (int)EndType.Win;
+                }
+                else if (Time.time - startTime >= paramCon.timeLimit)
+                {
+                    // lose
+                    //thisReward = paramCon.loseReward;
+                    thisReward = 0f;
+                    endTypeInt = (int)EndType.Lose;
+                }
+                else
+                {
+                    // keep on keeping on!
+                    endTypeInt = (int)EndType.Running;
+                }
+                break;
+        }
+        envUICon.showResult(endTypeInt);
+        worldUICon.updateChart(targetTypeInt, endTypeInt);
+        return (endTypeInt, thisReward);
+    }
+
+    // calculate kill reward base on killed enemy's position
+    public float killReward(Vector3 enemyPosition)
+    {
+        float thisKillReward = 0f;
+        if (targetTypeInt == (int)Targets.Attack)
+        {
+            // attack
+            (_, int isInArea) = blockCont.thisBlock.getDist_inArea(enemyPosition);
+            if (isInArea == 1)
+            {
+                thisKillReward = paramCon.killInAreaEnemyReward;
+            }
+            else
+            {
+                thisKillReward = paramCon.killReward;
+            }
+        }
+        else
+        {
+            // goto & defence & free
+            thisKillReward = paramCon.killReward;
+        }
+        return thisKillReward;
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 92a93d5d33962bc44b9fd2368358d47e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -0,0 +1,68 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using XCharts.Runtime;
+
+public class WorldUIController : MonoBehaviour
+{
+    public LineChart WinChart;
+    public int[] totalGames;
+    public int[] winGames;
+
+    private int maxXAxis = 0;
+    // Start is called before the first frame update
+    void Start()
+    {
+        totalGames = new int[(int)TargetController.Targets.Num];
+        winGames = new int[(int)TargetController.Targets.Num];
+        Array.Clear(totalGames, 0, (int)TargetController.Targets.Num);
+        Array.Clear(winGames, 0, (int)TargetController.Targets.Num);
+        //WinChart.Init();
+        WinChart.RemoveData();
+        for (int i = 0; i < (int)TargetController.Targets.Num; i++)
+        {
+            string lineName = Enum.GetName(typeof(TargetController.Targets), i);
+            WinChart.AddSerie<Line>(lineName);
+        }
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+
+    }
+    public void updateChart(int targetType, int endType)
+    {
+        float winRatio = 0f;
+        switch (endType)
+        {
+            case (int)TargetController.EndType.Win:
+                //Win
+                totalGames[targetType] += 1;
+                winGames[targetType] += 1;
+                winRatio = (float)winGames[targetType] / totalGames[targetType];
+                WinChart.AddData(targetType, winRatio);
+                if (totalGames[targetType] > maxXAxis)
+                {
+                    maxXAxis = totalGames[targetType];
+                    WinChart.AddXAxisData(Convert.ToString(maxXAxis));
+                }
+                break;
+            case (int)TargetController.EndType.Lose:
+                //lose
+                totalGames[targetType] += 1;
+                winRatio = (float)winGames[targetType] / totalGames[targetType];
+                WinChart.AddData(targetType, winRatio);
+                if (totalGames[targetType] > maxXAxis)
+                {
+                    maxXAxis = totalGames[targetType];
+                    WinChart.AddXAxisData(Convert.ToString(maxXAxis));
+                }
+                break;
+            default:
+                break;
+        }
+
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: fe230edb81d2a59409af4d033c8d2e7a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -0,0 +1,58 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public class states : MonoBehaviour
+{
+    public bool isDead = false;
+    public float MaxHP = 100;
+    float myHP = 100;
+
+    void Start()
+    {
+        myHP = MaxHP;
+    }
+
+    void Update()
+    {
+    }
+    private void detactDeath()
+    {
+        if (myHP <= 0)
+        {
+            Destroy(this.gameObject);
+            isDead = true;
+        }
+    }
+    // while got hit
+    public void ReactToHit(float Damage, GameObject damageSource)
+    {
+        myHP -= Damage;
+        Debug.Log("HP:" + myHP);
+        if (myHP <= 0)
+        {
+            if (damageSource.tag == "Player")
+            {
+                damageSource.GetComponent<AgentWithGun>().killRecord(transform.position);
+                Destroy(this.gameObject);
+                isDead = true;
+            }
+            else
+            {
+                Destroy(this.gameObject);
+                isDead = true;
+            }
+        }
+    }
+    // get my hp from other script
+    public float getnowHP()
+    {
+        return myHP;
+    }
+
+    public void destroyMe()
+    {
+        Destroy(this.gameObject);
+        isDead = true;
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b25389b3cd1e7084d81fa752823ef210
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: