Koha9/Aimbot-PPO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Save model weights immediately future

Browse Source 
add save button@Unity then send "saveNow" Toggle to Python.
delete useless method "saveModel" and "loadModel". Use Save/load weights instead.
This commit is contained in:
Koha9 2022-09-05 21:22:34 +09:00
parent 2d404cfdf2
commit 763d704efd
7 changed files with 202 additions and 527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Aimbot-PPO-MultiScene/Assets/ML-Agents/Timers/InGame_timers.json
View File

@ -1 +1 @@
{"count":1,"self":16.1291616,"total":16.1316623,"children":{"InitializeActuators":{"count":2,"self":0.0010018,"total":0.0010018,"children":null},"InitializeSensors":{"count":2,"self":0.001,"total":0.001,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1662370082","unity_version":"2020.3.19f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.19f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-PPO-MultiScene -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-UCUNI -hubSessionId 209fdf30-2c1f-11ed-916f-33e85f4223cc -accessToken 3CjC0epfWRwxG_KrVpNvr61CG1zT0gOQYZxf1vrmOrM00ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0","scene_name":"InGame","end_time_seconds":"1662370098"}}
{"count":1,"self":49.843855999999995,"total":49.849858,"children":{"InitializeActuators":{"count":2,"self":0.0010003,"total":0.0010003,"children":null},"InitializeSensors":{"count":2,"self":0.0010000999999999999,"total":0.0010000999999999999,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1662380026","unity_version":"2020.3.19f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2020.3.19f1\\Editor\\Unity.exe -projectpath C:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-PPO\\Aimbot-PPO-MultiScene -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-UCUNI -hubSessionId 209fdf30-2c1f-11ed-916f-33e85f4223cc -accessToken IuORxC7h70N41GHOqk3KOoetERc0UUQjx5TXQ7GcoN400ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.0","scene_name":"InGame","end_time_seconds":"1662380076"}}

287
Aimbot-PPO-MultiScene/Assets/Scenes/InGame.unity
View File

File diff suppressed because it is too large Load Diff

2
Aimbot-PPO-MultiScene/Assets/Script/AgentWithGun.cs
View File

@ -499,6 +499,8 @@ public class AgentWithGun : Agent
//sensor.AddObservation(raySensorNum); // raySensor数量 int
sensor.AddObservation(LoadDirDateF); // 用于loadModel的第一级dir
sensor.AddObservation(loadDirTimeF); // 用于loadModel的第二级dir
sensor.AddObservation(saveNow); // sent saveNow Toggle to python let agent save weights
saveNow = 0; // reset saveNow Toggle
//sensor.AddObservation(remainTime); // RemainTime int
}

2
Aimbot-PPO-MultiScene/Assets/Script/SaveWeightsButton.cs
View File

@ -7,5 +7,7 @@ public class SaveWeightsButton : MonoBehaviour
public GameObject Agent;
public void onButtonClicked()
{
// set saveNow to active.
Agent.GetComponent<AgentWithGun>().saveNow = 1;
}
}

48
Aimbot-PPO-Python/PPO.py
View File

@ -392,21 +392,22 @@ class PPO(object):
print(his.history['loss'])
return np.mean(his.history['loss'])
def saveWeights(self,score):
def saveWeights(self,score = None):
"""save now NN's Weight. Use "models.save_weights" method.
Save as "tf" format "ckpt" file.
Args:
score (int): now score
"""
actor_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
critic_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
score_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
self.actor.save_weights(actor_save_dir, save_format="tf")
self.critic.save_weights(critic_save_dir, save_format="tf")
if score != None:
# create an empty file named as score to recored score
score_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
scorefile = open(score_dir,'w')
scorefile.close()
actor_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
critic_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
self.actor.save_weights(actor_save_dir, save_format="tf")
self.critic.save_weights(critic_save_dir, save_format="tf")
print("Model's Weights Saved")
def loadWeightToModels(self,loadDir):
@ -425,38 +426,3 @@ class PPO(object):
print("++++++++++++Model Loaded++++++++++++")
print(loadDir)
print("++++++++++++++++++++++++++++++++++++")
def saveModel(self, score):
"""save now NN Model. Use "model.save()" method.
Args:
score (int): now score
"""
score = "_" + str(round(score))
actor_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + \
score+"/actor.h5"
critic_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + \
score+"/critic.h5"
self.actor.save(actor_save_dir)
self.critic.save(critic_save_dir)
print("Model Saved")
def loadModel(self, loadDir):
"""load NN Model. Use "models.load_model()" method.
Args:
loadDir (string): Model dir
Returns:
tf.keras.models: retuen compiled models.
"""
actorDir = loadDir+"/actor.h5"
criticDir = loadDir+"/critic.h5"
actor_net_loaded = tf.keras.models.load_model(actorDir)
critic_net_loaded = tf.keras.models.load_model(criticDir)
print("++++++++++++++++++++++++++++++++++++")
print("++++++++++++Model Loaded++++++++++++")
print(loadDir)
print("++++++++++++++++++++++++++++++++++++")
return actor_net_loaded, critic_net_loaded

22
Aimbot-PPO-Python/aimBotEnv.py
View File

@ -10,7 +10,7 @@ class makeEnv(object):
self.env.reset()
# get enviroment specs
self.LOAD_DIR_SIZE_IN_STATE = 2
self.LOAD_DIR_SIZE_IN_STATE = 3
self.TRACKED_AGENT = -1
self.BEHA_SPECS = self.env.behavior_specs
self.BEHA_NAME = list(self.BEHA_SPECS)[0]
@ -58,8 +58,8 @@ class makeEnv(object):
self.env.set_actions(behavior_name=behaviorName,action=thisActionTuple)
self.env.step()
# get nextState & reward & done after this action
nextState,reward,done,loadDir = self.getSteps(behaviorName,trackedAgent)
return nextState,reward,done,loadDir
nextState,reward,done,loadDir, saveNow = self.getSteps(behaviorName,trackedAgent)
return nextState,reward,done,loadDir, saveNow
def getSteps(self,behaviorName = None,trackedAgent = None):
# get nextState & reward & done
@ -74,23 +74,25 @@ class makeEnv(object):
if trackedAgent in decisionSteps: # ゲーム終了していない場合、環境状態がdecision_stepsに保存される
nextState = decisionSteps[trackedAgent].obs[0]
nextState = np.reshape(nextState,[1,self.STATE_SIZE+self.LOAD_DIR_SIZE_IN_STATE])
loadDir = nextState[0][-2:]
nextState = nextState[0][:-2]
saveNow = nextState[0][-1]
loadDir = nextState[0][-3:-1]
nextState = nextState[0][:-3]
reward = decisionSteps[trackedAgent].reward
done = False
if trackedAgent in terminalSteps: # ゲーム終了した場合、環境状態がterminal_stepsに保存される
nextState = terminalSteps[trackedAgent].obs[0]
nextState = np.reshape(nextState,[1,self.STATE_SIZE+self.LOAD_DIR_SIZE_IN_STATE])
loadDir = nextState[0][-2:]
nextState = nextState[0][:-2]
saveNow = nextState[0][-1]
loadDir = nextState[0][-3:-1]
nextState = nextState[0][:-3]
reward = terminalSteps[trackedAgent].reward
done = True
return nextState, reward, done, loadDir
return nextState, reward, done, loadDir, saveNow
def reset(self):
self.env.reset()
nextState,reward,done,loadDir = self.getSteps()
return nextState,reward,done,loadDir
nextState,reward,done,loadDir,saveNow = self.getSteps()
return nextState,reward,done,loadDir,saveNow
def render(self):
self.env.render()

368
Aimbot-PPO-Python/main.ipynb
View File

@ -68,8 +68,7 @@
"text": [
"√√√√√Enviroment Initialized Success√√√√√\n",
"√√√√√Buffer Initialized Success√√√√√\n",
"Load Model:\n",
"PPO-Model/09051833/183540\n",
"No loadDir specified,Create a New Model\n",
"CONTINUOUS_SIZE 1\n",
"DISCRETE_SIZE 5\n",
"STATE_SIZE 29\n"
@ -86,7 +85,7 @@
"STATE_SIZE = env.STATE_SIZE\n",
"CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n",
"DISCRETE_SIZE = env.DISCRETE_SIZE\n",
"s,reward,done,loadDir = env.getSteps()\n",
"_,_,_,loadDir,_ = env.getSteps()\n",
"\n",
"# check load model or not\n",
"if(np.any(loadDir == 0)):\n",
@ -160,11 +159,7 @@
"output_type": "stream",
"text": [
"EP 0 START\n",
"√√√√√Buffer Initialized Success√√√√√\n",
"++++++++++++++++++++++++++++++++++++\n",
"++++++++++++Model Loaded++++++++++++\n",
"PPO-Model/09051833/183540\n",
"++++++++++++++++++++++++++++++++++++\n"
"√√√√√Buffer Initialized Success√√√√√\n"
]
},
{
@ -181,337 +176,25 @@
"name": "stdout",
"output_type": "stream",
"text": [
"A_Loss: -43.132581075032554 C_Loss: 4025.360986328125\n",
"EP OVER!\n",
"A_Loss: 1184751.5479166666 C_Loss: 1939213379930.6667\n",
"SAVENOW!\n",
"Model's Weights Saved\n",
"New Record! Save NN -37.67000053450465\n",
"EP 1 START\n",
"A_Loss: 14251923.066666666 C_Loss: 40720630.7288086\n",
"EP OVER!\n",
"A_Loss: 1857006.7333333334 C_Loss: 245.15654500325522\n",
"A_Loss: 9210259745450.666 C_Loss: 7842064320569890.0\n",
"SAVENOW!\n",
"Model's Weights Saved\n",
"New Record! Save NN -13.100000601261854\n",
"EP 2 START\n",
"A_Loss: -1.0014899969100952 C_Loss: 71.29023424784343\n",
"EP OVER!\n",
"A_Loss: -0.31318608721097313 C_Loss: 8.897234630584716\n",
"EP 3 START\n",
"A_Loss: 566053.9979166667 C_Loss: 1986013.3489705403\n",
"EP OVER!\n",
"A_Loss: 0.6021817366282145 C_Loss: 6.035458342234294\n",
"EP 4 START\n",
"A_Loss: 169656457.6 C_Loss: 4317.756831359863\n",
"EP OVER!\n",
"A_Loss: 22155934.0 C_Loss: 63.52870483398438\n",
"EP 5 START\n",
"A_Loss: 0.2413090835014979 C_Loss: 10.40585823059082\n",
"EP OVER!\n",
"A_Loss: 0.12157159547011058 C_Loss: 14.347647285461425\n",
"EP 6 START\n",
"A_Loss: 215977770.66666666 C_Loss: 190435277.5966268\n",
"EP OVER!\n",
"A_Loss: 213349568.0 C_Loss: 1620.733740234375\n",
"EP 7 START\n",
"A_Loss: 100149825.6 C_Loss: 1569803.3794799806\n",
"EP OVER!\n",
"A_Loss: 171062395.73333332 C_Loss: 34872400594.933334\n",
"Model's Weights Saved\n",
"New Record! Save NN 4.119999349117279\n",
"EP 8 START\n",
"A_Loss: 0.7038820425669352 C_Loss: 31.980174128214518\n",
"EP OVER!\n",
"A_Loss: 0.2687960028648376 C_Loss: 6.94128991762797\n",
"EP 9 START\n",
"A_Loss: 0.1451285809278488 C_Loss: 3.5754743576049806\n",
"EP OVER!\n",
"A_Loss: 0.18081151247024535 C_Loss: 3.902424764633179\n",
"EP 10 START\n",
"A_Loss: 29139865.6 C_Loss: 174927458.90618488\n",
"EP OVER!\n",
"A_Loss: 16942552.4 C_Loss: 134.21190592447917\n",
"EP 11 START\n",
"A_Loss: 53790223.46666667 C_Loss: 647.0605305989583\n",
"EP OVER!\n",
"A_Loss: 27695839.866666667 C_Loss: 343.9340047200521\n",
"EP 12 START\n",
"A_Loss: 65695720.8 C_Loss: 61225647615.0198\n",
"EP OVER!\n",
"A_Loss: 0.07575627962748209 C_Loss: 10.41986296971639\n",
"EP 13 START\n",
"A_Loss: 51573392.266666666 C_Loss: 212022811.82775268\n",
"EP OVER!\n",
"A_Loss: 22073133.866666667 C_Loss: 61.18818028767904\n",
"EP 14 START\n",
"A_Loss: 0.13529965033133826 C_Loss: 0.966444210211436\n",
"EP OVER!\n",
"A_Loss: 0.1176165262858073 C_Loss: 0.5141626675923665\n",
"EP 15 START\n",
"A_Loss: 0.5104357699553171 C_Loss: 25.79277165730794\n",
"EP OVER!\n",
"A_Loss: 15181718.0 C_Loss: 9111658759031.467\n",
"EP 16 START\n",
"A_Loss: 30694314.533333335 C_Loss: 62842582.26665497\n",
"EP OVER!\n",
"A_Loss: 17702280.066666666 C_Loss: 50.18375091552734\n",
"EP 17 START\n",
"A_Loss: 0.08570613314708074 C_Loss: 1.7101642807324728\n",
"EP OVER!\n",
"A_Loss: 14154208.066666666 C_Loss: 248.65523885091145\n",
"EP 18 START\n",
"A_Loss: 51043696.8 C_Loss: 39972927.18101196\n",
"EP OVER!\n",
"A_Loss: 24850118.0 C_Loss: 6779725.618615722\n",
"EP 19 START\n",
"A_Loss: 75697662.4 C_Loss: 505.26551717122396\n",
"EP OVER!\n",
"A_Loss: 64376180.266666666 C_Loss: 299.2391876220703\n",
"EP 20 START\n",
"A_Loss: 55954064.8 C_Loss: 234.9048828125\n",
"EP OVER!\n",
"A_Loss: 57271699.733333334 C_Loss: 59254958.60783488\n",
"EP 21 START\n",
"A_Loss: 124046376.53333333 C_Loss: 2737571204155.7334\n",
"EP OVER!\n",
"A_Loss: 31774753.866666667 C_Loss: 24325611279.066666\n",
"EP 22 START\n",
"A_Loss: 7490019498.666667 C_Loss: 245487346653.86667\n",
"EP OVER!\n",
"A_Loss: 186999600.0 C_Loss: 36627312369.066666\n",
"EP 23 START\n",
"A_Loss: 4297551769.6 C_Loss: 1184376194184.5334\n",
"EP OVER!\n",
"A_Loss: 122982910044842.67 C_Loss: 7331505971746.134\n",
"EP 24 START\n",
"A_Loss: 2682021.3583333334 C_Loss: 2341072196027.7334\n",
"EP OVER!\n",
"A_Loss: 1469103463901.8667 C_Loss: 113593101.86666666\n",
"EP 25 START\n",
"A_Loss: 76648460.0 C_Loss: 14498072.481510418\n",
"EP OVER!\n",
"A_Loss: 5800877.766666667 C_Loss: 22134.297265625\n",
"EP 26 START\n",
"A_Loss: 34429532.53333333 C_Loss: 46926.630208333336\n",
"EP OVER!\n",
"A_Loss: 212612493.86666667 C_Loss: 23176.368880208334\n",
"EP 27 START\n",
"A_Loss: 3207813239.4666667 C_Loss: 41575.61041666667\n",
"EP OVER!\n",
"A_Loss: 38858107.86666667 C_Loss: 15486.2171875\n",
"EP 28 START\n",
"A_Loss: -58850.48046875 C_Loss: 14550529378106.69\n",
"EP OVER!\n",
"A_Loss: 445133260.8 C_Loss: 1749447703.4604166\n",
"EP 29 START\n",
"A_Loss: 80796092.8 C_Loss: 8523564.864453126\n",
"EP OVER!\n",
"A_Loss: 34027046.266666666 C_Loss: 3319.6830078125\n",
"Model's Weights Saved\n",
"New Record! Save NN 4.589999713003635\n",
"EP 30 START\n",
"A_Loss: 118412868.26666667 C_Loss: 23715976.025211588\n",
"EP OVER!\n",
"A_Loss: 94927646.93333334 C_Loss: 1067.18818359375\n",
"EP 31 START\n",
"A_Loss: 338770097.06666666 C_Loss: 3585200.7686075848\n",
"EP OVER!\n",
"A_Loss: 1479514146.1333334 C_Loss: 10348.122916666667\n",
"Model's Weights Saved\n",
"New Record! Save NN 38.5099995136261\n",
"EP 32 START\n",
"A_Loss: 119593341.86666666 C_Loss: 784.9417297363282\n",
"EP OVER!\n",
"A_Loss: -3.2093143463134766 C_Loss: 180.5671844482422\n",
"EP 33 START\n",
"A_Loss: 149599549.86666667 C_Loss: 206.16288146972656\n",
"EP OVER!\n",
"A_Loss: 5682631.7 C_Loss: 17.62857920328776\n",
"EP 34 START\n",
"A_Loss: 231213670.4 C_Loss: 272.8009948730469\n",
"EP OVER!\n",
"A_Loss: 167307104.0 C_Loss: 188.52051798502603\n",
"EP 35 START\n",
"A_Loss: 204825578.66666666 C_Loss: 335.75029296875\n",
"EP OVER!\n",
"A_Loss: 83991501.33333333 C_Loss: 451.3644246419271\n",
"EP 36 START\n",
"A_Loss: 0.6501724402109782 C_Loss: 38.280514017740884\n",
"EP OVER!\n",
"A_Loss: 175042737.06666666 C_Loss: 256.6915344238281\n",
"EP 37 START\n",
"A_Loss: 461190880.0 C_Loss: 717.4720499674479\n",
"EP OVER!\n",
"A_Loss: 173152186.66666666 C_Loss: 701.6123962402344\n",
"EP 38 START\n",
"A_Loss: 188070140.8 C_Loss: 1611377609664.9072\n",
"EP OVER!\n",
"A_Loss: 512005579391093.7 C_Loss: 86553492717568.0\n",
"EP 39 START\n",
"A_Loss: 121553.57962239583 C_Loss: 53618558110.88125\n",
"EP OVER!\n",
"A_Loss: 10892.0080078125 C_Loss: 132429.940625\n",
"EP 40 START\n",
"A_Loss: 15404.601106770833 C_Loss: 107106.1421875\n",
"EP OVER!\n",
"A_Loss: 1822.5163004557292 C_Loss: 182031.73854166668\n",
"EP 41 START\n",
"A_Loss: 599.1890706380208 C_Loss: 76118.35729166666\n",
"EP OVER!\n",
"A_Loss: 1033.1336588541667 C_Loss: 54971.41354166667\n",
"EP 42 START\n",
"A_Loss: 2602.054020182292 C_Loss: 227568.73541666666\n",
"EP OVER!\n",
"A_Loss: -3.2474422613779703 C_Loss: 155.22516682942708\n",
"EP 43 START\n",
"A_Loss: 1994.366455078125 C_Loss: 149748.48125\n",
"EP OVER!\n",
"A_Loss: 7441.857747395833 C_Loss: 196344.43958333333\n",
"EP 44 START\n",
"A_Loss: 19441.808723958333 C_Loss: 197749.58020833333\n",
"EP OVER!\n",
"A_Loss: 1789.1592529296875 C_Loss: 25795.180598958334\n",
"EP 45 START\n",
"A_Loss: 6888.085221354167 C_Loss: 149084.575\n",
"EP OVER!\n",
"A_Loss: 1102350.757421875 C_Loss: 180759.19166666668\n",
"EP 46 START\n",
"A_Loss: 536.078184000651 C_Loss: 89909.24375\n",
"EP OVER!\n",
"A_Loss: 794341992174.4 C_Loss: 59005630871824.9\n",
"EP 47 START\n",
"A_Loss: 114184075127.46666 C_Loss: 74263479521553.06\n",
"EP OVER!\n",
"A_Loss: 1293861572.3333333 C_Loss: 1474176015837.8667\n",
"EP 48 START\n",
"A_Loss: 1143.1249674479166 C_Loss: 19221068817.411232\n",
"EP OVER!\n",
"A_Loss: 855.1484252929688 C_Loss: 28676.431770833333\n",
"EP 49 START\n",
"A_Loss: 1198.7060546875 C_Loss: 41875.48359375\n",
"EP OVER!\n",
"A_Loss: 1215.0390218098958 C_Loss: 7167.594791666666\n",
"EP 50 START\n",
"A_Loss: 1840.496435546875 C_Loss: 33346.76875\n",
"EP OVER!\n",
"A_Loss: 97.68925425211589 C_Loss: 2453.3792643229167\n",
"EP 51 START\n",
"A_Loss: 2179.5446614583334 C_Loss: 25573.869270833333\n",
"EP OVER!\n",
"A_Loss: 3547.454052734375 C_Loss: 15107.698307291666\n",
"EP 52 START\n",
"A_Loss: 513.1327128092448 C_Loss: 23233.316276041667\n",
"EP OVER!\n",
"A_Loss: 2024.9339111328125 C_Loss: 10457.81875\n",
"EP 53 START\n",
"A_Loss: 321.8419494628906 C_Loss: 10830.605924479167\n",
"EP OVER!\n",
"A_Loss: -8.38951981862386 C_Loss: 2241.023612467448\n",
"EP 54 START\n",
"A_Loss: 1454.901765950521 C_Loss: 16337.481966145833\n",
"EP OVER!\n",
"A_Loss: 3069.5313313802085 C_Loss: 4450.756380208333\n",
"EP 55 START\n",
"A_Loss: 444.8317138671875 C_Loss: 15201.106184895832\n",
"EP OVER!\n",
"A_Loss: 1159.2985107421875 C_Loss: 35525.82317708333\n",
"EP 56 START\n",
"A_Loss: 1397.7694742838542 C_Loss: 8600.898404947917\n",
"EP OVER!\n",
"A_Loss: 160.49285074869792 C_Loss: 3749.308203125\n",
"EP 57 START\n",
"A_Loss: -23.67707913716634 C_Loss: 8787.21162109375\n",
"EP OVER!\n",
"A_Loss: 446.2368998209635 C_Loss: 8949.849674479166\n",
"EP 58 START\n",
"A_Loss: 1443.8223063151042 C_Loss: 4806.931754557291\n",
"EP OVER!\n",
"A_Loss: 1028.3776000976563 C_Loss: 10080.125455729167\n",
"EP 59 START\n",
"A_Loss: 979.55380859375 C_Loss: 18168.035872395834\n",
"EP OVER!\n",
"A_Loss: 2799.885953776042 C_Loss: 14584.458203125\n",
"EP 60 START\n",
"A_Loss: 83247.30833333333 C_Loss: 61687910677571.76\n",
"EP OVER!\n",
"A_Loss: 81.43709259033203 C_Loss: 47630220.59791667\n",
"EP 61 START\n",
"A_Loss: 5032.870377604167 C_Loss: 43944.79453125\n",
"EP OVER!\n",
"A_Loss: 9.13815720876058 C_Loss: 2573.8714192708335\n",
"EP 62 START\n",
"A_Loss: 523.7818908691406 C_Loss: 8441.000618489583\n",
"EP OVER!\n",
"A_Loss: 1593.0845377604167 C_Loss: 17598.201953125\n",
"EP 63 START\n",
"A_Loss: 27.476329803466797 C_Loss: 5594.57646484375\n",
"EP OVER!\n",
"A_Loss: 25.40858777364095 C_Loss: 11878.195572916668\n",
"EP 64 START\n",
"A_Loss: 3347.620393880208 C_Loss: 15555.9111328125\n",
"EP OVER!\n",
"A_Loss: 25.35328992207845 C_Loss: 7923.337727864583\n",
"EP 65 START\n",
"A_Loss: 1800.478165690104 C_Loss: 36845.799479166664\n",
"EP OVER!\n",
"A_Loss: 544.9222574869792 C_Loss: 2949.7829752604166\n",
"EP 66 START\n",
"A_Loss: 238.4148183186849 C_Loss: 2108.7259195963543\n",
"EP OVER!\n",
"A_Loss: 10.808652464548747 C_Loss: 1069.8261800130208\n",
"EP 67 START\n",
"A_Loss: 7.2742944399515785 C_Loss: 1112.0956298828125\n",
"EP OVER!\n",
"A_Loss: 686.372725423177 C_Loss: 1421.539111328125\n",
"EP 68 START\n",
"A_Loss: 38.04718589782715 C_Loss: 2771.959724934896\n",
"EP OVER!\n",
"A_Loss: 1133.577986653646 C_Loss: 5758.061686197917\n",
"EP 69 START\n",
"A_Loss: 402.1093465169271 C_Loss: 3987.9388671875\n",
"EP OVER!\n",
"A_Loss: -1.7969589630762737 C_Loss: 2884.10732421875\n",
"EP 70 START\n",
"A_Loss: 1016.8031656901042 C_Loss: 6636.073665364584\n",
"EP OVER!\n",
"A_Loss: 939.9079630533854 C_Loss: 3692.443391927083\n",
"EP 71 START\n",
"A_Loss: -107.65687510172526 C_Loss: 21740540498583.367\n",
"EP OVER!\n",
"A_Loss: 5876.485221354166 C_Loss: 50534.828385416666\n",
"EP 72 START\n",
"A_Loss: 219.98338216145834 C_Loss: 2713.9967041015625\n",
"EP OVER!\n",
"A_Loss: 433.9121032714844 C_Loss: 950.6825520833333\n",
"EP 73 START\n",
"A_Loss: 933.4014811197917 C_Loss: 1297.3184733072917\n",
"EP OVER!\n",
"A_Loss: 2035.8281412760416 C_Loss: 3957.7666178385416\n",
"EP 74 START\n",
"A_Loss: 479.3147298177083 C_Loss: 1727.3513753255208\n",
"EP OVER!\n",
"A_Loss: 579.0808024088542 C_Loss: 635.3917744954427\n",
"EP 75 START\n",
"A_Loss: 1450.3585856119792 C_Loss: 2148.2808756510417\n",
"EP OVER!\n",
"A_Loss: 917.6628540039062 C_Loss: 1192.1792277018228\n",
"EP 76 START\n",
"A_Loss: 888.0876180013021 C_Loss: 1050.4912679036458\n",
"EP OVER!\n",
"A_Loss: 249.14984232584635 C_Loss: 9789.355598958333\n",
"EP 77 START\n",
"A_Loss: 346.7358866373698 C_Loss: 1828.4739990234375\n",
"EP OVER!\n",
"A_Loss: 15.01647071838379 C_Loss: 1649.2421305338542\n",
"EP 78 START\n",
"A_Loss: 2058.658439127604 C_Loss: 1996.608447265625\n",
"EP OVER!\n",
"A_Loss: 155.17740580240886 C_Loss: 575.6137990315755\n",
"EP 79 START\n",
"A_Loss: 1246.8288167317708 C_Loss: 1408.468212890625\n",
"EP OVER!\n",
"A_Loss: 175.76889241536458 C_Loss: 825.604911295573\n",
"EP 80 START\n",
"A_Loss: 10138501.666666666 C_Loss: 33048526138440.633\n"
"A_Loss: 4103941.316666667 C_Loss: 410607418692949.3\n"
]
},
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: 'PPO-Model/09052116/211645/-53'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_15440/420232317.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 86\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mepTotalReward\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mmaxTotalReward\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mepTotalReward\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 87\u001b[0m \u001b[0mmaxTotalReward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mepTotalReward\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 88\u001b[1;33m \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msaveWeights\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepTotalReward\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 89\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"New Record! Save NN\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepTotalReward\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 90\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32mc:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-PPO\\Aimbot-PPO-Python\\PPO.py\u001b[0m in \u001b[0;36msaveWeights\u001b[1;34m(self, score)\u001b[0m\n\u001b[0;32m 403\u001b[0m \u001b[1;31m# create an empty file named as score to recored score\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 404\u001b[0m \u001b[0mscore_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msaveDir\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"%H%M%S\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"/\"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 405\u001b[1;33m \u001b[0mscorefile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscore_dir\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'w'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 406\u001b[0m \u001b[0mscorefile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 407\u001b[0m \u001b[0mactor_save_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msaveDir\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"%H%M%S\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"/actor/\"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"actor.ckpt\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'PPO-Model/09052116/211645/-53'"
]
}
],
@ -528,7 +211,7 @@
"for ep in range(MAX_EP):\n",
" print(\"EP \",ep,\" START\")\n",
" # first time run game\n",
" s,_,_,_ = env.reset()\n",
" s,_,_,_,_ = env.reset()\n",
" if (ep == 0):\n",
" epBuffer = buffer.buffer()\n",
" s = s.reshape([STATE_SIZE])\n",
@ -557,6 +240,9 @@
" epActorLossHis = []\n",
" epCriticLossHis = []\n",
" \n",
" # save weight immediately?\n",
" saveNow = 0;\n",
"\n",
" while not done:\n",
" step += 1\n",
" if step % ACTION_INTERVAL == 0: # take action every ACTION_INTERVAL steps\n",
@ -564,7 +250,7 @@
" disAct1,disAct2,disAct3,conAct,predictResult = agent.chooseAction(s)\n",
" kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(disAct1,disAct2,disAct3,conAct)\n",
" \n",
" nextState,thisReward,done,_ = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n",
" nextState,thisReward,done,_,saveNow = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n",
"\n",
" epTotalReward += thisReward\n",
" epBuffer.saveBuffers(s,[disAct1,disAct2,disAct3,conAct],thisReward)\n",
@ -572,13 +258,17 @@
" disActs = np.array([[0,0,0,0,0]])\n",
" conActs = np.array([[0]])\n",
"\n",
" nextState,thisReward,done,_ = env.step(discreteActions=disActs,continuousActions=conActs)\n",
" nextState,thisReward,done,_,saveNow = env.step(discreteActions=disActs,continuousActions=conActs)\n",
" epTotalReward += thisReward\n",
" nextState = nextState.reshape([STATE_SIZE])\n",
" s = nextState\n",
" \n",
" if done:\n",
" print(\"EP OVER!\")\n",
" if saveNow != 0:\n",
" print(\"SAVENOW!\")\n",
" saveNow = 0\n",
" agent.saveWeights()\n",
" # update PPO after Batch step or GameOver\n",
" if (step+1)%BATCH == 0 or done:\n",
" bs = epBuffer.getStates()\n",