Update PPO class,add python human control
Python: Update PPO class add python human control Unity: add FP/TP choose button
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"√√√√√Enviroment Initialized Success√√√√√\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import time\n",
|
||||
"import aimBotEnv\n",
|
||||
"from HumanAction import HumanActions\n",
|
||||
"\n",
|
||||
"# Env\n",
|
||||
"ENV_PATH = \"./Build-CloseEnemyCut/Aimbot-PPO\"\n",
|
||||
"WORKER_ID = 1\n",
|
||||
"BASE_PORT = 200\n",
|
||||
"\n",
|
||||
"MOUSEDISCOUNT = 8.0\n",
|
||||
"MAX_EP = 10000000\n",
|
||||
"\n",
|
||||
"env = aimBotEnv.makeEnv(envPath=ENV_PATH, workerID=WORKER_ID, basePort=BASE_PORT)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "UnityCommunicatorStoppedException",
|
||||
"evalue": "Communicator has exited.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mUnityCommunicatorStoppedException\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_37248/645561173.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 6\u001b[0m \u001b[0mactions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdemoAct\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetHumanActions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mactions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
|
||||
"\u001b[1;32mc:\\Users\\UCUNI\\OneDrive\\Unity\\ML-Agents\\Aimbot-PPO\\Aimbot-PPO-Python\\aimBotEnv.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, actions, behaviorName, trackedAgent)\u001b[0m\n\u001b[0;32m 72\u001b[0m \u001b[1;31m# take action to env\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 73\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_actions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbehavior_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbehaviorName\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maction\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mthisActionTuple\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 75\u001b[0m \u001b[1;31m# get nextState & reward & done after this action\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 76\u001b[0m \u001b[0mnextState\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloadDir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaveNow\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetSteps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbehaviorName\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrackedAgent\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[1;32mc:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\mlagents_envs\\timers.py\u001b[0m in \u001b[0;36mwrapped\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 303\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 304\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mhierarchical_timer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__qualname__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 305\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 306\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 307\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m \u001b[1;31m# type: ignore\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[1;32mc:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\mlagents_envs\\environment.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 333\u001b[0m \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_communicator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexchange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep_input\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_poll_process\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 334\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0moutputs\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 335\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mUnityCommunicatorStoppedException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Communicator has exited.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 336\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_update_behavior_specs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 337\u001b[0m \u001b[0mrl_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrl_output\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[1;31mUnityCommunicatorStoppedException\u001b[0m: Communicator has exited."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"done = False\n",
|
||||
"env.reset()\n",
|
||||
"demoAct = HumanActions(mouseDiscount=MOUSEDISCOUNT)\n",
|
||||
"for ep in range(MAX_EP):\n",
|
||||
" while not done:\n",
|
||||
" actions = demoAct.getHumanActions()\n",
|
||||
" env.step(actions=actions)6\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.7 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.7"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import keyboard
|
||||
import mouse
|
||||
|
||||
|
||||
class HumanActions:
|
||||
def __init__(self, mouseDiscount: float = 10, screenW: int = 1920, screenH: int = 1080):
|
||||
def multiPressed():
|
||||
pass
|
||||
|
||||
keyboard.add_hotkey("w+a", multiPressed)
|
||||
keyboard.add_hotkey("w+d", multiPressed)
|
||||
keyboard.add_hotkey("s+a", multiPressed)
|
||||
keyboard.add_hotkey("s+d", multiPressed)
|
||||
self.screenW = screenW
|
||||
self.screenH = screenH
|
||||
self.MOUSEDISCOUNT = mouseDiscount
|
||||
|
||||
def getHumanActions(self):
|
||||
x, _ = mouse.get_position()
|
||||
xMovement = (x - self.screenW / 2) / self.MOUSEDISCOUNT
|
||||
|
||||
ws = 0
|
||||
ad = 0
|
||||
click = 0
|
||||
if keyboard.is_pressed("w"):
|
||||
ws = 1
|
||||
elif keyboard.is_pressed("s"):
|
||||
ws = 2
|
||||
if keyboard.is_pressed("d"):
|
||||
ad = 1
|
||||
elif keyboard.is_pressed("a"):
|
||||
ad = 2
|
||||
if keyboard.is_pressed("w+d"):
|
||||
ws = 1
|
||||
ad = 1
|
||||
elif keyboard.is_pressed("w+a"):
|
||||
ws = 1
|
||||
ad = 2
|
||||
elif keyboard.is_pressed("s+d"):
|
||||
ws = 2
|
||||
ad = 1
|
||||
elif keyboard.is_pressed("s+a"):
|
||||
ws = 2
|
||||
ad = 2
|
||||
if mouse.is_pressed(button="left"):
|
||||
click = 1
|
||||
|
||||
actions = [ws, ad, click, [xMovement]]
|
||||
|
||||
mouse.move(self.screenW / 2, self.screenH / 2)
|
||||
return actions
|
||||
File diff suppressed because it is too large
Load Diff
+556
-269
@@ -1,108 +1,213 @@
|
||||
import tensorflow as tf
|
||||
from tensorflow.python.ops.numpy_ops import ndarray
|
||||
import tensorflow_probability as tfp
|
||||
import numpy as np
|
||||
import time
|
||||
import math
|
||||
import copy
|
||||
|
||||
import datetime
|
||||
import os
|
||||
from PPOConfig import PPOConfig
|
||||
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
from tensorflow.keras import optimizers
|
||||
from keras_radam import RAdam
|
||||
|
||||
EPS = 1e-10
|
||||
|
||||
|
||||
class PPO(object):
|
||||
"""Create PPO Agent
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
stateSize: int,
|
||||
disActShape: list,
|
||||
conActSize: int,
|
||||
conActRange: float,
|
||||
PPOConfig: PPOConfig,
|
||||
):
|
||||
"""initialize PPO
|
||||
|
||||
Args:
|
||||
stateSize (int): enviroment state size
|
||||
disActShape (numpy): discrete Action shape.
|
||||
just like [3,2],means 2 type of dis actions,each act include 3 and 2 types
|
||||
if no discrete action output then use [0].
|
||||
conActSize (int): continuous Action Size. if no continuous action output then use 0.
|
||||
conActRange (float): continuous action range. -conActRange to +conActRange
|
||||
PPOConfig (PPOConfig): PPO configuration
|
||||
"""
|
||||
# check use dis action or not.
|
||||
if disActShape == [0]:
|
||||
# non dis action output
|
||||
self.disActSize = 0
|
||||
self.disOutputSize = 0
|
||||
else:
|
||||
# make sure disActShape greater than 1
|
||||
try:
|
||||
if np.any(np.array(disActShape) <= 1):
|
||||
raise ValueError(
|
||||
"disActShape error,disActShape should greater than 1 but get", disActShape
|
||||
)
|
||||
except ValueError:
|
||||
raise
|
||||
self.disActSize = len(disActShape)
|
||||
self.disOutputSize = sum(disActShape)
|
||||
|
||||
def __init__(self, stateSize, disActShape, conActSize, conActRange, criticLR, actorLR, gamma, epsilon, entropyWeight, saveDir, loadModelDir):
|
||||
|
||||
# check disActShape is correct(greater than 1)
|
||||
try:
|
||||
if np.any(np.array(disActShape)<=1):
|
||||
raise ValueError("disActShape error,disActShape should greater than 1 but get",disActShape)
|
||||
except ValueError as e:
|
||||
raise
|
||||
|
||||
self.stateSize = stateSize
|
||||
# self.actionSize = actionSize
|
||||
self.disActShape = disActShape # shape of discrete action output. like [3,3,2]
|
||||
self.disActSize = len(disActShape)
|
||||
self.disActShape = disActShape
|
||||
self.conActSize = conActSize
|
||||
self.conActRange = conActRange
|
||||
self.criticLR = criticLR
|
||||
self.actorLR = actorLR
|
||||
self.GAMMA = gamma
|
||||
self.EPSILON = epsilon
|
||||
self.saveDir = saveDir
|
||||
self.entropyWeight = entropyWeight
|
||||
self.muSigSize = 2
|
||||
self.conOutputSize = conActSize * self.muSigSize
|
||||
|
||||
self.disOutputSize = sum(disActShape)
|
||||
self.conOutputSize = conActSize * 2
|
||||
|
||||
if loadModelDir == None:
|
||||
# config
|
||||
self.NNShape = PPOConfig.NNShape
|
||||
self.criticLR = PPOConfig.criticLR
|
||||
self.actorLR = PPOConfig.actorLR
|
||||
self.gamma = PPOConfig.gamma
|
||||
self.lmbda = PPOConfig.lmbda
|
||||
self.clipRange = PPOConfig.clipRange
|
||||
self.entropyWeight = PPOConfig.entropyWeight
|
||||
self.trainEpochs = PPOConfig.trainEpochs
|
||||
self.saveDir = PPOConfig.saveDir
|
||||
self.loadModelDir = PPOConfig.loadModelDir
|
||||
print("---------thisPPO Params---------")
|
||||
print("self.stateSize = ", self.stateSize)
|
||||
print("self.disActShape = ", self.disActShape)
|
||||
print("self.disActSize", self.disActSize)
|
||||
print("self.disOutputSize", self.disOutputSize)
|
||||
print("self.conActSize = ", self.conActSize)
|
||||
print("self.conActRange = ", self.conActRange)
|
||||
print("self.conOutputSize = ", self.conOutputSize)
|
||||
|
||||
# config
|
||||
print("---------thisPPO config---------")
|
||||
print("self.NNShape = ", self.NNShape)
|
||||
print("self.criticLR = ", self.criticLR)
|
||||
print("self.actorLR = ", self.actorLR)
|
||||
print("self.gamma = ", self.gamma)
|
||||
print("self.lmbda = ", self.lmbda)
|
||||
print("self.clipRange = ", self.clipRange)
|
||||
print("self.entropyWeight = ", self.entropyWeight)
|
||||
print("self.trainEpochs = ", self.trainEpochs)
|
||||
print("self.saveDir = ", self.saveDir)
|
||||
print("self.loadModelDir = ", self.loadModelDir)
|
||||
|
||||
# load NN or not
|
||||
if self.loadModelDir is None:
|
||||
# critc NN
|
||||
self.critic = self.buildCriticNet(self.stateSize, 1, compileModel = True)
|
||||
self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
|
||||
# actor NN
|
||||
self.actor = self.buildActorNet(self.stateSize, self.conActRange, compileModel = True)
|
||||
self.actor = self.buildActorNet(self.stateSize, compileModel=True)
|
||||
print("---------Actor Model Create Success---------")
|
||||
self.actor.summary()
|
||||
print("---------Critic Model Create Success---------")
|
||||
self.critic.summary()
|
||||
else:
|
||||
# critc NN
|
||||
self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
|
||||
# actor NN
|
||||
self.actor = self.buildActorNet(self.stateSize, self.conActRange, compileModel=True)
|
||||
self.actor = self.buildActorNet(self.stateSize, compileModel=True)
|
||||
# load weight to Critic&Actor NN
|
||||
self.loadWeightToModels(loadModelDir)
|
||||
|
||||
self.loadWeightToModels(self.loadModelDir)
|
||||
print("---------Actor Model Load Success---------")
|
||||
self.actor.summary()
|
||||
print("---------Critic Model Load Success---------")
|
||||
self.critic.summary()
|
||||
|
||||
# Build Net
|
||||
def buildActorNet(self, inputSize, continuousActionRange,compileModel):
|
||||
def buildActorNet(self, inputSize: int, compileModel: bool):
|
||||
"""build Actor Nueral Net and compile.Output:[disAct1,disAct2,disAct3,mu,sigma]
|
||||
|
||||
Args:
|
||||
inputSize (int): InputLayer Nueral size.
|
||||
continuousActionRange (foat): continuous Action's max Range.
|
||||
compileModel (bool): compile Model or not.
|
||||
|
||||
Returns:
|
||||
keras.Model: return Actor NN
|
||||
"""
|
||||
stateInput = layers.Input(shape=(inputSize,), name='stateInput')
|
||||
dense0 = layers.Dense(500, activation='relu',name='dense0',)(stateInput)
|
||||
dense1 = layers.Dense(200, activation='relu',name='dense1',)(dense0)
|
||||
dense2 = layers.Dense(100, activation='relu', name='dense2')(dense1)
|
||||
# -----------Input Layers-----------
|
||||
stateInput = layers.Input(shape=(inputSize,), name="stateInput")
|
||||
|
||||
disAct1 = layers.Dense(3, activation='softmax',name='WSAction')(dense2) # WS
|
||||
disAct2 = layers.Dense(3, activation='softmax',name='ADAction')(dense2) # AD
|
||||
disAct3 = layers.Dense(2, activation='softmax',name='ShootAction')(dense2) # Mouse shoot
|
||||
mu = continuousActionRange * layers.Dense(1, activation='tanh', name='muOut')(dense2) # mu，既正态分布mean
|
||||
sigma = 1e-8 + layers.Dense(1, activation='softplus',name='sigmaOut')(dense2) # sigma，既正态分布
|
||||
# musig = layers.concatenate([mu,sigma],name = 'musig')
|
||||
totalOut = layers.concatenate(
|
||||
[disAct1, disAct2, disAct3, mu, sigma], name='totalOut') # package
|
||||
# -------Intermediate layers--------
|
||||
interLayers = []
|
||||
interLayersIndex = 0
|
||||
for neuralUnit in self.NNShape:
|
||||
thisLayerName = "dense" + str(interLayersIndex)
|
||||
if interLayersIndex == 0:
|
||||
interLayers.append(
|
||||
layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
|
||||
)
|
||||
else:
|
||||
interLayers.append(
|
||||
layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
|
||||
)
|
||||
interLayersIndex += 1
|
||||
|
||||
# ----------Output Layers-----------
|
||||
outputLayersList = []
|
||||
if self.disActSize != 0:
|
||||
# while NN have discrete action output.
|
||||
disActIndex = 0
|
||||
for thisDisActDepth in self.disActShape:
|
||||
thisDisActName = "disAct" + str(disActIndex)
|
||||
outputLayersList.append(
|
||||
layers.Dense(thisDisActDepth, activation="softmax", name=thisDisActName)(
|
||||
interLayers[-1]
|
||||
)
|
||||
)
|
||||
disActIndex += 1
|
||||
if self.conActSize != 0:
|
||||
# while NN have continuous action output.
|
||||
mu = tf.multiply(
|
||||
layers.Dense(1, activation="tanh", name="muOut")(interLayers[-1]), self.conActRange
|
||||
) # mu，既正态分布位置参数
|
||||
sigma = tf.add(
|
||||
layers.Dense(1, activation="softplus", name="sigmaOut")(interLayers[-1]), EPS
|
||||
) # sigma，既正态分布尺度参数
|
||||
outputLayersList.append(mu)
|
||||
outputLayersList.append(sigma)
|
||||
totalOut = layers.concatenate(outputLayersList, name="totalOut") # package
|
||||
|
||||
# ----------Model Compile-----------
|
||||
model = keras.Model(inputs=stateInput, outputs=totalOut)
|
||||
#actorOPT = optimizers.Adam(learning_rate = self.actorLR)
|
||||
if compileModel:
|
||||
actorOPT = RAdam(self.actorLR)
|
||||
if compileModel: # Compile Model
|
||||
actorOPT = optimizers.Adam(learning_rate=self.actorLR)
|
||||
model.compile(optimizer=actorOPT, loss=self.aLoss())
|
||||
return model
|
||||
|
||||
def buildCriticNet(self, inputSize, outputSize,compileModel):
|
||||
def buildCriticNet(self, inputSize: int, outputSize: int, compileModel: bool):
|
||||
"""build Critic Nueral Net and compile.Output:[Q]
|
||||
|
||||
Args:
|
||||
inputSize (int): InputLayer Neural Size
|
||||
outputSize (float): Q size
|
||||
inputSize (int): input size
|
||||
outputSize (int): output size
|
||||
compileModel (bool): compile Model or not.
|
||||
|
||||
Returns:
|
||||
keras.Model: return Critic NN
|
||||
"""
|
||||
stateInput = keras.Input(shape=(inputSize,))
|
||||
dense0 = layers.Dense(500, activation='relu',
|
||||
name='dense0',)(stateInput)
|
||||
dense1 = layers.Dense(200, activation='relu')(dense0)
|
||||
dense2 = layers.Dense(100, activation='relu')(dense1)
|
||||
output = layers.Dense(outputSize)(dense2)
|
||||
# -----------Input Layers-----------
|
||||
stateInput = keras.Input(shape=(inputSize,), name="stateInput")
|
||||
|
||||
# -------Intermediate layers--------
|
||||
interLayers = []
|
||||
interLayersIndex = 0
|
||||
for neuralUnit in self.NNShape:
|
||||
thisLayerName = "dense" + str(interLayersIndex)
|
||||
if interLayersIndex == 0:
|
||||
interLayers.append(
|
||||
layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
|
||||
)
|
||||
else:
|
||||
interLayers.append(
|
||||
layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
|
||||
)
|
||||
interLayersIndex += 1
|
||||
|
||||
# ----------Output Layers-----------
|
||||
output = layers.Dense(outputSize, activation=None)(interLayers[-1])
|
||||
|
||||
# ----------Model Compile-----------
|
||||
model = keras.Model(inputs=stateInput, outputs=output)
|
||||
if compileModel:
|
||||
criticOPT = optimizers.Adam(learning_rate=self.criticLR)
|
||||
@@ -110,39 +215,53 @@ class PPO(object):
|
||||
return model
|
||||
|
||||
# loss Function
|
||||
# critic loss
|
||||
def cLoss(self):
|
||||
"""Critic Loss function
|
||||
"""
|
||||
"""Critic Loss function"""
|
||||
|
||||
def loss(y_true, y_pred):
|
||||
# y_true: discountedR
|
||||
# y_pred: critcV = model.predict(states)
|
||||
|
||||
advantage = y_true - y_pred # TD error
|
||||
loss = tf.reduce_mean(tf.square(advantage))
|
||||
adv = y_true - y_pred # TD error
|
||||
loss = tf.reduce_mean(tf.square(adv))
|
||||
return loss
|
||||
|
||||
return loss
|
||||
|
||||
# actor loss
|
||||
def aLoss(self):
|
||||
def getDiscreteALoss(nowProbs,oldProbs,advantage):
|
||||
"""Actor Loss function"""
|
||||
|
||||
def getDiscreteALoss(nowProbs, oldProbs, disOneHotAct, actShape, advantage):
|
||||
"""get Discrete Action Loss
|
||||
|
||||
Args:
|
||||
nowProbs (tf.constant): (length,actionSize)
|
||||
oldProbs (tf.constant): (length,actionSize)
|
||||
nowProbs (tf.constant): (length,actionProbSize)
|
||||
oldProbs (tf.constant): (length,actionProbSize)
|
||||
advantage (tf.constant): (length,)
|
||||
|
||||
Returns:
|
||||
tf.constant: (length,)
|
||||
"""
|
||||
entropy = tf.reduce_mean(tf.math.multiply(nowProbs,tf.math.log(nowProbs+1e-6)))
|
||||
ratio = tf.math.divide(nowProbs,oldProbs+1e-6)
|
||||
value = tf.math.multiply(ratio,tf.expand_dims(advantage,axis = 1))
|
||||
clipRatio = tf.clip_by_value(ratio,1. - self.EPSILON,1.+self.EPSILON)
|
||||
clipValue = tf.math.multiply(clipRatio,tf.expand_dims(advantage,axis = 1))
|
||||
loss = -tf.reduce_mean(tf.math.minimum(value,clipValue)) + self.entropyWeight * entropy
|
||||
entropy = tf.negative(
|
||||
tf.reduce_mean(tf.math.multiply(nowProbs, tf.math.log(nowProbs + EPS)))
|
||||
)
|
||||
nowSingleProbs = tf.reduce_mean(tf.multiply(nowProbs, disOneHotAct), axis=1)
|
||||
nowSingleProbs = tf.multiply(nowSingleProbs, actShape)
|
||||
oldSingleProbs = tf.reduce_mean(tf.multiply(oldProbs, disOneHotAct), axis=1)
|
||||
oldSingleProbs = tf.multiply(oldSingleProbs, actShape)
|
||||
ratio = tf.math.divide(nowSingleProbs, oldSingleProbs + EPS)
|
||||
value = tf.math.multiply(ratio, advantage)
|
||||
clipRatio = tf.clip_by_value(ratio, 1.0 - self.clipRange, 1.0 + self.clipRange)
|
||||
clipValue = tf.math.multiply(clipRatio, advantage)
|
||||
loss = tf.math.negative(
|
||||
tf.reduce_mean(tf.math.minimum(value, clipValue))
|
||||
- tf.multiply(self.entropyWeight, entropy)
|
||||
)
|
||||
return loss
|
||||
|
||||
def getContinuousALoss(musig,actions,oldProbs,advantage):
|
||||
|
||||
def getContinuousALoss(musig, actions, oldProbs, advantage):
|
||||
"""get Continuous Action Loss
|
||||
|
||||
Args:
|
||||
@@ -154,150 +273,376 @@ class PPO(object):
|
||||
Returns:
|
||||
tf.constant: (length,)
|
||||
"""
|
||||
mu = musig[:,0]
|
||||
sigma = musig[:,1]
|
||||
dist = tfp.distributions.Normal(mu,sigma)
|
||||
|
||||
mu = musig[:, 0]
|
||||
sigma = musig[:, 1]
|
||||
dist = tfp.distributions.Normal(mu, sigma)
|
||||
|
||||
nowProbs = dist.prob(actions)
|
||||
ratio = tf.math.divide(nowProbs,oldProbs+1e-6)
|
||||
entropy = tf.reduce_mean(dist.entropy())
|
||||
|
||||
value = tf.math.multiply(ratio,tf.expand_dims(advantage,axis = 1))
|
||||
clipValue = tf.clip_by_value(ratio,1. - self.EPSILON,1.+self.EPSILON) * advantage
|
||||
loss = -tf.reduce_mean(tf.math.minimum(value,clipValue)) + self.entropyWeight * entropy
|
||||
|
||||
ratio = tf.math.divide(nowProbs, oldProbs + EPS)
|
||||
value = tf.math.multiply(ratio, advantage)
|
||||
clipRatio = tf.clip_by_value(ratio, 1.0 - self.clipRange, 1.0 + self.clipRange)
|
||||
clipValue = tf.math.multiply(clipRatio, advantage)
|
||||
loss = tf.negative(
|
||||
tf.reduce_mean(tf.math.minimum(value, clipValue))
|
||||
- tf.multiply(self.entropyWeight, entropy)
|
||||
)
|
||||
return loss
|
||||
|
||||
def loss(y_true, y_pred):
|
||||
# y_true: [[disAct1, disAct2, disAct3, mu, sigma]]
|
||||
# y_pred: muSigma = self.actor(state) =
|
||||
# [[disAct1, disAct2, disAct3, mu, sigma]]
|
||||
oldDisProbs = y_true[:,0:self.disOutputSize]
|
||||
oldConMusigs = y_true[:,self.disOutputSize:self.disOutputSize+self.conActSize]
|
||||
conActions = y_true[:,self.disOutputSize+self.conActSize:self.disOutputSize+(self.conActSize*2)]
|
||||
advantage = y_true[:,-1]
|
||||
|
||||
nowDisProbs = y_pred[:,0:self.disOutputSize] # [disAct1, disAct2, disAct3]
|
||||
nowConMusigs = y_pred[:,self.disOutputSize:] #[musig1,musig2]
|
||||
|
||||
totalALoss = tf.constant([0.])
|
||||
# y_true: [[disActProb..., conActProbs..., disOneHotActs..., conAct..., advantage]]
|
||||
# y_pred: [[disActProb..., mu, sigma...]]
|
||||
totalALoss = 0
|
||||
totalActionNum = 0
|
||||
|
||||
# for nowProb,oldProb in zip(tf.transpose(nowDisProbs,perm=[1,0,2]),tf.transpose(oldDisProbs,perm=[1,0,2])):
|
||||
lastDisActShape = 0
|
||||
for shape in self.disActShape:
|
||||
thisNowDisProbs = nowDisProbs[:,lastDisActShape:lastDisActShape+shape]
|
||||
thisOldDisProbs = oldDisProbs[:,lastDisActShape:lastDisActShape+shape]
|
||||
discreteALoss = getDiscreteALoss(thisNowDisProbs,thisOldDisProbs,advantage)
|
||||
lastDisActShape += shape
|
||||
totalALoss += discreteALoss
|
||||
totalActionNum += 1
|
||||
# for nowConMusig,conAction,oldPiProb in zip(tf.transpose(nowConMusigs,perm=[1,0,2]),conActions,oldPiProbs):
|
||||
lastConAct = 0
|
||||
for act in range(self.conActSize):
|
||||
thisNowConMusig = nowConMusigs[:,lastConAct:lastConAct+((act+1)*2)]
|
||||
thisOldConMusig = oldConMusigs[:,lastConAct:lastConAct+((act+1)*2)]
|
||||
thisConAction = conActions[:,act]
|
||||
continuousAloss = getContinuousALoss(thisNowConMusig,thisConAction,thisOldConMusig,advantage)
|
||||
totalALoss += continuousAloss
|
||||
totalActionNum += 1
|
||||
advantage = tf.expand_dims(y_true[:, -1], axis=1)
|
||||
|
||||
loss = tf.divide(totalALoss,totalActionNum)
|
||||
if self.disActSize != 0:
|
||||
# while NN have discrete action output.
|
||||
oldDisProbs = y_true[:, 0 : self.disOutputSize]
|
||||
nowDisProbs = y_pred[:, 0 : self.disOutputSize] # [disAct1, disAct2, disAct3]
|
||||
disOneHotActs = y_true[
|
||||
:,
|
||||
self.disOutputSize
|
||||
+ self.conActSize : self.disOutputSize
|
||||
+ self.conActSize
|
||||
+ self.disOutputSize,
|
||||
]
|
||||
lastDisActShape = 0
|
||||
for thisShape in self.disActShape:
|
||||
thisNowDisProbs = nowDisProbs[:, lastDisActShape : lastDisActShape + thisShape]
|
||||
thisOldDisProbs = oldDisProbs[:, lastDisActShape : lastDisActShape + thisShape]
|
||||
thisDisOneHotActs = disOneHotActs[
|
||||
:, lastDisActShape : lastDisActShape + thisShape
|
||||
]
|
||||
discreteALoss = getDiscreteALoss(
|
||||
thisNowDisProbs, thisOldDisProbs, thisDisOneHotActs, thisShape, advantage
|
||||
)
|
||||
lastDisActShape += thisShape
|
||||
totalALoss += discreteALoss
|
||||
totalActionNum += 1.0
|
||||
if self.conActSize != 0:
|
||||
# while NN have continuous action output.
|
||||
oldConProbs = y_true[:, self.disOutputSize : self.disOutputSize + self.conActSize]
|
||||
conActions = y_true[
|
||||
:,
|
||||
self.disOutputSize
|
||||
+ self.conActSize : self.disOutputSize
|
||||
+ self.conActSize
|
||||
+ self.conActSize,
|
||||
]
|
||||
nowConMusigs = y_pred[:, self.disOutputSize :] # [musig1,musig2]
|
||||
lastConAct = 0
|
||||
for conAct in range(self.conActSize):
|
||||
thisNowConMusig = nowConMusigs[:, lastConAct : lastConAct + self.muSigSize]
|
||||
thisOldConProb = oldConProbs[:, conAct : conAct + 1]
|
||||
thisConAction = conActions[:, conAct]
|
||||
continuousAloss = getContinuousALoss(
|
||||
thisNowConMusig, thisConAction, thisOldConProb, advantage
|
||||
)
|
||||
totalALoss += continuousAloss
|
||||
totalActionNum += 1.0
|
||||
lastConAct += self.muSigSize
|
||||
loss = tf.divide(totalALoss, totalActionNum)
|
||||
return loss
|
||||
|
||||
return loss
|
||||
|
||||
# get Action&V
|
||||
def chooseAction(self, state):
|
||||
# get Actions&values
|
||||
def chooseAction(self, state: ndarray):
|
||||
"""Agent choose action to take
|
||||
|
||||
Args:
|
||||
state (np.array): enviroment state
|
||||
state (ndarray): enviroment state
|
||||
|
||||
Returns:
|
||||
np.array:
|
||||
disAct1,
|
||||
discreteAction1
|
||||
disAct2,
|
||||
discreteAction2
|
||||
disAct3,
|
||||
discreteAction3
|
||||
conAction,
|
||||
continuousAction
|
||||
np.array:
|
||||
actions,
|
||||
actions list,2dims like [[0],[1],[1.5]]
|
||||
predictResult,
|
||||
actor NN predict Result output
|
||||
"""
|
||||
# let actor choose action,use the normal distribution
|
||||
# state = np.expand_dims(state,0)
|
||||
|
||||
# check state dimension is [1,statesize]
|
||||
if state.ndim!=2:
|
||||
state = state.reshape([1,self.stateSize])
|
||||
|
||||
|
||||
# check state dimension is [stateNum,statesize]
|
||||
if state.ndim != 2:
|
||||
stateNum = int(len(state) / self.stateSize)
|
||||
state = state.reshape([stateNum, self.stateSize])
|
||||
predictResult = self.actor(state) # get predict result [[disAct1, disAct2, disAct3, musig]]
|
||||
predictResult = predictResult.numpy()
|
||||
disAct1Prob = predictResult[0][0:3]
|
||||
disAct2Prob = predictResult[0][3:6]
|
||||
disAct3Prob = predictResult[0][6:8]
|
||||
mu = predictResult[0][8]
|
||||
sigma = predictResult[0][9]
|
||||
if math.isnan(mu) or math.isnan(sigma):
|
||||
# check mu or sigma is nan
|
||||
print("mu or sigma is nan")
|
||||
# print("predictResult",predictResult)
|
||||
# predictResult = predictResult.numpy()
|
||||
actions = []
|
||||
if self.disActSize != 0:
|
||||
# while NN have discrete action output.
|
||||
lastDisActShape = 0
|
||||
for shape in self.disActShape:
|
||||
thisDisActProbs = predictResult[:, lastDisActShape : lastDisActShape + shape]
|
||||
dist = tfp.distributions.Categorical(probs=thisDisActProbs, dtype=tf.float32)
|
||||
action = int(dist.sample().numpy()[0])
|
||||
# action = np.argmax(thisDisActProbs)
|
||||
actions.append(action)
|
||||
lastDisActShape += shape
|
||||
if self.conActSize != 0:
|
||||
# while NN have continuous action output.
|
||||
lastConAct = 0
|
||||
for actIndex in range(self.conActSize):
|
||||
thisMu = predictResult[:, self.disOutputSize + lastConAct]
|
||||
thisSig = predictResult[:, self.disOutputSize + lastConAct + 1]
|
||||
if math.isnan(thisMu) or math.isnan(thisSig):
|
||||
# check mu or sigma is nan
|
||||
print("chooseAction:mu or sigma is nan")
|
||||
thisDist = np.random.normal(loc=thisMu, scale=thisSig)
|
||||
actions.append(np.clip(thisDist, -self.conActRange, self.conActRange))
|
||||
lastConAct += 2
|
||||
return actions, predictResult
|
||||
|
||||
disAct1 = np.argmax(disAct1Prob) # WS 0 or 1 or 2
|
||||
disAct2 = np.argmax(disAct2Prob) # AD 0 or 1 or 2
|
||||
disAct3 = np.argmax(disAct3Prob) # mouse shoot 0 or 1
|
||||
normDist = np.random.normal(loc=mu, scale=sigma) # normalDistribution
|
||||
conAction = np.clip(normDist, -self.conActRange,
|
||||
self.conActRange) # 在正态分布中随机get一个action
|
||||
return disAct1, disAct2, disAct3, conAction, predictResult
|
||||
def trainCritcActor(
|
||||
self,
|
||||
states: ndarray,
|
||||
oldActorResult: ndarray,
|
||||
actions: ndarray,
|
||||
rewards: ndarray,
|
||||
dones: ndarray,
|
||||
nextState: ndarray,
|
||||
epochs: int = None,
|
||||
):
|
||||
"""train critic&actor use PPO ways
|
||||
|
||||
def getCriticV(self, state):
|
||||
Args:
|
||||
states (ndarray): states
|
||||
oldActorResult (ndarray): actor predict result
|
||||
actions (ndarray): predicted actions include both discrete actions and continuous actions
|
||||
rewards (ndarray): rewards from enviroment
|
||||
dones (ndarray): dones from enviroment
|
||||
nextState (ndarray): next state from enviroment
|
||||
epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.
|
||||
|
||||
Returns:
|
||||
tf.constant: criticLoss, actorLoss
|
||||
"""
|
||||
|
||||
if epochs == None:
|
||||
epochs = self.trainEpochs
|
||||
criticValues = self.getCriticV(state=states)
|
||||
discountedR = self.discountReward(nextState, criticValues, dones, rewards)
|
||||
advantage = self.getGAE(discountedR, criticValues)
|
||||
|
||||
criticLoss = self.trainCritic(states, discountedR, epochs)
|
||||
actorLoss = self.trainActor(states, oldActorResult, actions, advantage, epochs)
|
||||
# print("A_Loss:", actorLoss, "C_Loss:", criticLoss)
|
||||
return criticLoss, actorLoss
|
||||
|
||||
def trainCritic(self, states: ndarray, discountedR: ndarray, epochs: int = None):
|
||||
"""critic NN trainning function
|
||||
|
||||
Args:
|
||||
states (ndarray): states
|
||||
discountedR (ndarray): discounted rewards
|
||||
epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.
|
||||
|
||||
Returns:
|
||||
tf.constant: all critic losses
|
||||
"""
|
||||
if epochs == None:
|
||||
epochs = self.trainEpochs
|
||||
his = self.critic.fit(x=states, y=discountedR, epochs=epochs, verbose=0)
|
||||
return his.history["loss"]
|
||||
|
||||
def trainActor(
|
||||
self,
|
||||
states: ndarray,
|
||||
oldActorResult: ndarray,
|
||||
actions: ndarray,
|
||||
advantage: ndarray,
|
||||
epochs: int = None,
|
||||
):
|
||||
"""actor NN trainning function
|
||||
|
||||
Args:
|
||||
states (ndarray): states
|
||||
oldActorResult (ndarray): actor predict results
|
||||
actions (ndarray): acotor predict actions
|
||||
advantage (ndarray): GAE advantage
|
||||
epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.
|
||||
|
||||
Returns:
|
||||
tf.constant: all actor losses
|
||||
"""
|
||||
# Trian Actor
|
||||
# states: Buffer States
|
||||
# actions: Buffer Actions
|
||||
# discountedR: Discounted Rewards
|
||||
# Epochs: just Epochs
|
||||
if epochs == None:
|
||||
epochs = self.trainEpochs
|
||||
actions = np.asarray(actions, dtype=np.float32)
|
||||
|
||||
disActions = actions[:, 0 : self.disActSize]
|
||||
conActions = actions[:, self.disActSize :]
|
||||
oldDisProbs = oldActorResult[:, 0 : self.disOutputSize] # [disAct1, disAct2, disAct3]
|
||||
oldConMusigs = oldActorResult[:, self.disOutputSize :] # [musig1,musig2]
|
||||
if self.disActSize != 0:
|
||||
disOneHotActs = self.getOneHotActs(disActions)
|
||||
if self.conActSize != 0:
|
||||
# while NN have discrete6 & continuous actions output.
|
||||
oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
|
||||
# pack [oldDisProbs,oldPiProbs,conActions,advantage] as y_true
|
||||
y_true = np.hstack((oldDisProbs, oldPiProbs, disOneHotActs, conActions, advantage))
|
||||
else:
|
||||
# while NN have only discrete actions output.
|
||||
# pack [oldDisProbs,advantage] as y_true
|
||||
y_true = np.hstack((oldDisProbs, disOneHotActs, advantage))
|
||||
else:
|
||||
if self.conActSize != 0:
|
||||
# while NN have only continuous action output.
|
||||
oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
|
||||
# pack [oldPiProbs,conActions,advantage] as y_true
|
||||
y_true = np.hstack((oldPiProbs, conActions, advantage))
|
||||
else:
|
||||
print("trainActor:disActSize & conActSize error")
|
||||
time.sleep(999999)
|
||||
# assembly Actions history
|
||||
# train start
|
||||
if np.any(tf.math.is_nan(y_true)):
|
||||
print("y_true got nan")
|
||||
print("y_true", y_true)
|
||||
his = self.actor.fit(x=states, y=y_true, epochs=epochs, verbose=0)
|
||||
if np.any(tf.math.is_nan(his.history["loss"])):
|
||||
print("his.history['loss'] is nan!")
|
||||
print(his.history["loss"])
|
||||
return his.history["loss"]
|
||||
|
||||
def saveWeights(self, score: float):
|
||||
"""save now NN's Weight. Use "models.save_weights" method.
|
||||
Save as "tf" format "ckpt" file.
|
||||
|
||||
Args:
|
||||
score (float): now score
|
||||
"""
|
||||
actor_save_dir = (
|
||||
self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
|
||||
)
|
||||
critic_save_dir = (
|
||||
self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
|
||||
)
|
||||
self.actor.save_weights(actor_save_dir, save_format="tf")
|
||||
self.critic.save_weights(critic_save_dir, save_format="tf")
|
||||
# create an empty file named as score to recored score
|
||||
score_dir = (
|
||||
self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
|
||||
)
|
||||
scorefile = open(score_dir, "w")
|
||||
scorefile.close()
|
||||
print("Model's Weights Saved")
|
||||
|
||||
def loadWeightToModels(self, loadDir: str):
|
||||
"""load NN Model. Use "models.load_weights()" method.
|
||||
Load "tf" format "ckpt" file.
|
||||
|
||||
Args:
|
||||
loadDir (str): Model dir
|
||||
"""
|
||||
actorDir = loadDir + "/actor/" + "actor.ckpt"
|
||||
criticDir = loadDir + "/critic/" + "critic.ckpt"
|
||||
self.actor.load_weights(actorDir)
|
||||
self.critic.load_weights(criticDir)
|
||||
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
print("++++++++++++Model Loaded++++++++++++")
|
||||
print(loadDir)
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
|
||||
def getCriticV(self, state: ndarray):
|
||||
"""get Critic predict V value
|
||||
|
||||
Args:
|
||||
state (np.array): Env state
|
||||
state (ndarray): Env state
|
||||
|
||||
Returns:
|
||||
tensor: retrun Critic predict result
|
||||
"""
|
||||
# if state.ndim < 2:
|
||||
# state = np.expand_dims(state,0)
|
||||
if state.ndim!=2:
|
||||
state = state.reshape([1,self.stateSize])
|
||||
if state.ndim != 2:
|
||||
stateNum = int(len(state) / self.stateSize)
|
||||
state = state.reshape([stateNum, self.stateSize])
|
||||
return self.critic.predict(state)
|
||||
|
||||
def discountReward(self, nextState, rewards):
|
||||
def discountReward(self, nextState: ndarray, values: ndarray, dones: ndarray, rewards: ndarray):
|
||||
"""Discount future rewards
|
||||
|
||||
Args:
|
||||
nextState (np.array): next Env state
|
||||
rewards (np.array): reward list of this episode
|
||||
nextState (ndarray): next Env state
|
||||
values (ndarray): critic predict values
|
||||
dones (ndarray): dones from enviroment
|
||||
rewards (ndarray): reward list of this episode
|
||||
|
||||
Returns:
|
||||
np.array: discounted rewards list,same shape as rewards that input
|
||||
ndarray: discounted rewards list,same shape as rewards that input
|
||||
"""
|
||||
"""
|
||||
nextV = self.getCriticV(nextState)
|
||||
dones = 1 - dones
|
||||
discountedRewards = []
|
||||
for i in reversed(range(len(rewards))):
|
||||
nextV = rewards[i] + dones[i] * self.gamma * nextV
|
||||
discountedRewards.append(nextV)
|
||||
discountedRewards.reverse() # reverse
|
||||
discountedRewards = np.squeeze(discountedRewards)
|
||||
discountedRewards = np.expand_dims(discountedRewards, axis=1)
|
||||
# discountedRewards = np.array(discountedRewards)[:, np.newaxis]
|
||||
return discountedRewards
|
||||
"""
|
||||
"""
|
||||
# 降低未来的rewards
|
||||
nextV = self.getCriticV(nextState)
|
||||
discountedRewards = []
|
||||
for r in rewards[::-1]:
|
||||
nextV = r + self.GAMMA*nextV
|
||||
nextV = r + self.gamma * nextV
|
||||
discountedRewards.append(nextV)
|
||||
discountedRewards.reverse() # \ESREVER/
|
||||
discountedRewards.reverse() # reverse
|
||||
discountedRewards = np.squeeze(discountedRewards)
|
||||
discountedRewards = np.expand_dims(discountedRewards, axis=1)
|
||||
#discountedRewards = np.array(discountedRewards)[:, np.newaxis]
|
||||
# discountedRewards = np.array(discountedRewards)[:, np.newaxis]
|
||||
print(discountedRewards)
|
||||
return discountedRewards
|
||||
"""
|
||||
g = 0
|
||||
discountedRewards = []
|
||||
lastValue = self.getCriticV(nextState)
|
||||
values = np.append(values, lastValue, axis=0)
|
||||
dones = 1 - dones
|
||||
for i in reversed(range(len(rewards))):
|
||||
delta = rewards[i] + self.gamma * values[i + 1] * dones[i] - values[i]
|
||||
g = delta + self.gamma * self.lmbda * dones[i] * g
|
||||
discountedRewards.append(g + values[i])
|
||||
discountedRewards.reverse()
|
||||
return np.asarray(discountedRewards)
|
||||
|
||||
def conProb(self, mu, sig, x):
|
||||
def getGAE(self, discountedRewards: ndarray, values: ndarray):
|
||||
"""compute GAE adcantage
|
||||
|
||||
Args:
|
||||
discountedRewards (ndarray): discounted rewards
|
||||
values (ndarray): critic predict values
|
||||
|
||||
Returns:
|
||||
ndarray: GAE advantage
|
||||
"""
|
||||
advantage = discountedRewards - values
|
||||
advantage = (advantage - np.mean(advantage)) / (np.std(advantage) + EPS)
|
||||
return advantage
|
||||
|
||||
def conProb(self, mu: ndarray, sig: ndarray, x: ndarray):
|
||||
"""calculate probability when x in Normal distribution(mu,sigma)
|
||||
|
||||
Args:
|
||||
mu (np,array): mu
|
||||
sig (np.array): sigma
|
||||
x (np.array): x
|
||||
mu (ndarray): mu
|
||||
sig (ndarray): sigma
|
||||
x (ndarray): x
|
||||
|
||||
Returns:
|
||||
np.array: probabilities
|
||||
ndarray: probability
|
||||
"""
|
||||
# 获取在正态分布mu,sig下当取x值时的概率
|
||||
# return shape : (length,1)
|
||||
@@ -309,120 +654,62 @@ class PPO(object):
|
||||
prob = dist.prob(x)
|
||||
|
||||
prob = np.reshape(prob, (np.size(x), 1))
|
||||
#dist = 1./(tf.sqrt(2.*np.pi)*sig)
|
||||
#prob = dist*tf.exp(-tf.square(x-mu)/(2.*tf.square(sig)))
|
||||
# dist = 1./(tf.sqrt(2.*np.pi)*sig)
|
||||
# prob = dist*tf.exp(-tf.square(x-mu)/(2.*tf.square(sig)))
|
||||
return prob
|
||||
|
||||
def trainCritcActor(self, states, actions, rewards, nextState, criticEpochs, actorEpochs):
|
||||
# Train ActorNN and CriticNN
|
||||
# states: Buffer States
|
||||
# actions: Buffer Actions
|
||||
# rewards: Buffer Rewards,没有Discount处理
|
||||
# nextState: 下一个单独state
|
||||
# criticEpochs: just criticNN'Epochs
|
||||
# acotrEpochs: just acotrNN'Epochs
|
||||
discountedR = self.discountReward(nextState, rewards)
|
||||
|
||||
criticMeanLoss = self.trainCritic(states, discountedR, criticEpochs)
|
||||
actorMeanLoss = self.trainActor(
|
||||
states, actions, discountedR, actorEpochs)
|
||||
print("A_Loss:", actorMeanLoss, "C_Loss:", criticMeanLoss)
|
||||
return actorMeanLoss, criticMeanLoss
|
||||
|
||||
def trainCritic(self, states, discountedR, epochs):
|
||||
# Trian Critic
|
||||
# states: Buffer States
|
||||
# discountedR: Discounted Rewards
|
||||
# Epochs: just Epochs
|
||||
|
||||
# IDK why this should be list...It just work...
|
||||
# If discountR in np.array type it will throw 'Failed to find data adapter that can handle'
|
||||
# discountedR = discountedR.tolist()
|
||||
his = self.critic.fit(x=states, y=discountedR,
|
||||
epochs=epochs, verbose=0)
|
||||
return np.mean(his.history['loss'])
|
||||
|
||||
def trainActor(self, states, actions, discountedR, epochs):
|
||||
"""Actor NN trainning function
|
||||
def getOneHotActs(self, disActions):
|
||||
"""one hot action encoder
|
||||
|
||||
Args:
|
||||
states (np.array): Env states
|
||||
actions (np.array): action history
|
||||
discountedR (np.array): discountedR
|
||||
epochs (int): epochs,how many time NN learning
|
||||
disActions (ndarray): discrete actions
|
||||
|
||||
Returns:
|
||||
Average actor loss: this learning round's average actor loss
|
||||
ndarray: one hot actions
|
||||
"""
|
||||
# Trian Actor
|
||||
# states: Buffer States
|
||||
# actions: Buffer Actions
|
||||
# discountedR: Discounted Rewards
|
||||
# Epochs: just Epochs
|
||||
actIndex = 0
|
||||
for thisShape in self.disActShape:
|
||||
thisActs = disActions[:, actIndex]
|
||||
thisOneHotAct = tf.squeeze(tf.one_hot(thisActs, thisShape)).numpy()
|
||||
if actIndex == 0:
|
||||
oneHotActs = thisOneHotAct
|
||||
else:
|
||||
oneHotActs = np.append(oneHotActs, thisOneHotAct, axis=1)
|
||||
actIndex += 1
|
||||
return oneHotActs
|
||||
|
||||
states = np.asarray(states)
|
||||
actions = np.asarray(actions, dtype=np.float32)
|
||||
# predict with old Actor NN
|
||||
oldActorResult = self.actor.predict(states)
|
||||
|
||||
# assembly Actions history
|
||||
disActions = actions[:,0:self.disActSize]
|
||||
conActions = actions[:,self.disActSize:]
|
||||
# assembly predictResult as old Actor's Result
|
||||
oldDisProbs = oldActorResult[:,0:self.disOutputSize] # [disAct1, disAct2, disAct3]
|
||||
oldConMusigs = oldActorResult[:,self.disOutputSize:] # [musig1,musig2]
|
||||
oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
|
||||
|
||||
criticV = self.critic.predict(states)
|
||||
advantage = copy.deepcopy(discountedR - criticV)
|
||||
|
||||
# pack [oldDisProbs,oldPiProbs,conActions,advantage] as y_true
|
||||
y_true = np.hstack((oldDisProbs,oldPiProbs,conActions,advantage))
|
||||
|
||||
# train start
|
||||
if np.any(tf.math.is_nan(y_true)):
|
||||
print("y_true got nan")
|
||||
print("oldConMusigs",oldConMusigs)
|
||||
print("oldPiProbs",oldPiProbs)
|
||||
print("conActions",conActions)
|
||||
print("oldConMusigs",oldConMusigs)
|
||||
his = self.actor.fit(x=states, y=y_true, epochs=epochs, verbose=0)
|
||||
if np.any(tf.math.is_nan(his.history['loss'])):
|
||||
print("his.history['loss'] is nan!")
|
||||
print(his.history['loss'])
|
||||
return np.mean(his.history['loss'])
|
||||
|
||||
def saveWeights(self,score = None):
|
||||
"""save now NN's Weight. Use "models.save_weights" method.
|
||||
Save as "tf" format "ckpt" file.
|
||||
def getAverageEntropy(self, probs: ndarray):
|
||||
"""get average dis&con ACT Entropys
|
||||
|
||||
Args:
|
||||
score (int): now score
|
||||
"""
|
||||
actor_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
|
||||
critic_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
|
||||
self.actor.save_weights(actor_save_dir, save_format="tf")
|
||||
self.critic.save_weights(critic_save_dir, save_format="tf")
|
||||
if score != None:
|
||||
# create an empty file named as score to recored score
|
||||
score_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
|
||||
scorefile = open(score_dir,'w')
|
||||
scorefile.close()
|
||||
print("Model's Weights Saved")
|
||||
|
||||
def loadWeightToModels(self,loadDir):
|
||||
"""load NN Model. Use "models.load_weights()" method.
|
||||
Load "tf" format "ckpt" file.
|
||||
probs (ndarray): actor NN predict result
|
||||
|
||||
Args:
|
||||
loadDir (string): Model dir
|
||||
Returns:
|
||||
float: average total entropy
|
||||
list: discrete entropys
|
||||
list: continuous entropys
|
||||
"""
|
||||
actorDir = loadDir + "/actor/" + "actor.ckpt"
|
||||
criticDir = loadDir + "/critic/" + "critic.ckpt"
|
||||
self.actor.load_weights(actorDir)
|
||||
self.critic.load_weights(criticDir)
|
||||
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
print("++++++++++++Model Loaded++++++++++++")
|
||||
print(loadDir)
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
discreteEntropys = []
|
||||
continuousEntropys = []
|
||||
if self.disActSize != 0:
|
||||
disProbs = probs[:, 0 : self.disOutputSize]
|
||||
lastDisActIndex = 0
|
||||
for actShape in self.disActShape:
|
||||
thisDisProbs = disProbs[:, lastDisActIndex : lastDisActIndex + actShape]
|
||||
lastDisActIndex += actShape
|
||||
discreteEntropys.append(
|
||||
tf.negative(
|
||||
tf.reduce_mean(
|
||||
tf.math.multiply(thisDisProbs, tf.math.log(thisDisProbs + EPS))
|
||||
)
|
||||
)
|
||||
)
|
||||
if self.conActSize != 0:
|
||||
conProbs = probs[:, self.disOutputSize :]
|
||||
conActIndex = 0
|
||||
for i in range(self.conActSize):
|
||||
thisConProbs = conProbs[:, conActIndex : conActIndex + 2]
|
||||
conActIndex += 2
|
||||
continuousEntropys.append(tf.reduce_mean(thisConProbs[:, 1]))
|
||||
averageEntropy = np.mean([np.mean(discreteEntropys), np.mean(continuousEntropys)])
|
||||
return averageEntropy, discreteEntropys, continuousEntropys
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class PPOBuffer(object):
|
||||
def __init__(self):
|
||||
self.states = []
|
||||
self.actorProbs = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
self.dones = []
|
||||
print("√√√√√Buffer Initialized Success√√√√√")
|
||||
|
||||
def clearBuffer(self):
|
||||
self.states = []
|
||||
self.actorProbs = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
self.dones = []
|
||||
|
||||
def getStates(self):
|
||||
return self.standDims(np.asarray(self.states))
|
||||
|
||||
def getActorProbs(self):
|
||||
return self.standDims(np.asarray(self.actorProbs))
|
||||
|
||||
def getActions(self):
|
||||
return self.standDims(np.asarray(self.actions))
|
||||
|
||||
def getRewards(self):
|
||||
return self.standDims(np.asarray(self.rewards))
|
||||
|
||||
def getDones(self):
|
||||
return self.standDims(np.asarray(self.dones))
|
||||
|
||||
def saveState(self, state):
|
||||
self.states.append(state)
|
||||
|
||||
def saveAction(self, action):
|
||||
self.actions.append(action)
|
||||
|
||||
def saveReward(self, reward):
|
||||
self.rewards.append(reward)
|
||||
|
||||
def standDims(self, data):
|
||||
# standarlize data's dimension
|
||||
if np.ndim(data) > 2:
|
||||
return np.squeeze(data, axis=1)
|
||||
elif np.ndim(data) < 2:
|
||||
return np.expand_dims(data, axis=1)
|
||||
else:
|
||||
return np.asarray(data)
|
||||
|
||||
def saveBuffers(self, state, actorProb, action, reward, done):
|
||||
self.states.append(state)
|
||||
self.actorProbs.append(actorProb)
|
||||
self.actions.append(action)
|
||||
self.rewards.append(reward)
|
||||
self.dones.append(done)
|
||||
"""
|
||||
print("self.states", self.states)
|
||||
print("self.actions", self.actions)
|
||||
print("self.rewards", self.rewards)
|
||||
print("self.dones", self.dones)
|
||||
print("self.values", self.values)
|
||||
"""
|
||||
@@ -0,0 +1,15 @@
|
||||
import datetime
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
|
||||
class PPOConfig(NamedTuple):
|
||||
NNShape: list = [256, 256, 128]
|
||||
actorLR: float = 2e-3 # Actor Net Learning
|
||||
criticLR: float = 2e-3 # Critic Net Learning
|
||||
gamma: float = 0.99
|
||||
lmbda: float = 0.95
|
||||
clipRange: float = 0.20
|
||||
entropyWeight: float = 1e-2
|
||||
trainEpochs: int = 8
|
||||
saveDir: str = "PPO-Model/" + datetime.datetime.now().strftime("%m%d-%H%M") + "/"
|
||||
loadModelDir: Optional[str] = None
|
||||
@@ -0,0 +1,58 @@
|
||||
from turtle import color
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
class PPOHistory(object):
|
||||
def __init__(self):
|
||||
self.meanRewards = []
|
||||
self.entropys = []
|
||||
self.actorLosses = []
|
||||
self.criticLosses = []
|
||||
|
||||
def saveHis(self, rewards, entropys, aLosses, cLosses):
|
||||
self.meanRewards.extend([rewards])
|
||||
self.entropys.extend([entropys])
|
||||
self.actorLosses.extend(aLosses)
|
||||
self.criticLosses.extend(cLosses)
|
||||
|
||||
def drawHis(self):
|
||||
plt.figure(figsize=(21, 13), facecolor="#011627")
|
||||
ax = plt.subplot(2, 2, 1)
|
||||
ax.set_facecolor("#011627")
|
||||
ax.spines["top"].set_color("#c9d2df")
|
||||
ax.spines["bottom"].set_color("#c9d2df")
|
||||
ax.spines["left"].set_color("#c9d2df")
|
||||
ax.spines["right"].set_color("#c9d2df")
|
||||
ax.plot(
|
||||
range(len(self.meanRewards)), self.meanRewards, color="#c9d2df", label="AverageRewards"
|
||||
)
|
||||
ax.set_title("meanRewards", color="#c9d2df")
|
||||
ax = plt.subplot(2, 2, 2)
|
||||
ax.set_facecolor("#011627")
|
||||
ax.spines["top"].set_color("#c9d2df")
|
||||
ax.spines["bottom"].set_color("#c9d2df")
|
||||
ax.spines["left"].set_color("#c9d2df")
|
||||
ax.spines["right"].set_color("#c9d2df")
|
||||
ax.plot(range(len(self.entropys)), self.entropys, color="#c9d2df", label="AverageEntropys")
|
||||
ax.set_title("entropys", color="#c9d2df")
|
||||
ax = plt.subplot(2, 2, 3)
|
||||
ax.set_facecolor("#011627")
|
||||
ax.spines["top"].set_color("#c9d2df")
|
||||
ax.spines["bottom"].set_color("#c9d2df")
|
||||
ax.spines["left"].set_color("#c9d2df")
|
||||
ax.spines["right"].set_color("#c9d2df")
|
||||
ax.plot(
|
||||
range(len(self.actorLosses)), self.actorLosses, color="#c9d2df", label="actorLosses"
|
||||
)
|
||||
ax.set_title("actorLosses", color="#c9d2df")
|
||||
ax = plt.subplot(2, 2, 4)
|
||||
ax.set_facecolor("#011627")
|
||||
ax.spines["top"].set_color("#c9d2df")
|
||||
ax.spines["bottom"].set_color("#c9d2df")
|
||||
ax.spines["left"].set_color("#c9d2df")
|
||||
ax.spines["right"].set_color("#c9d2df")
|
||||
ax.plot(
|
||||
range(len(self.criticLosses)), self.criticLosses, color="#c9d2df", label="criticLosses"
|
||||
)
|
||||
ax.set_title("criticLosses", color="#c9d2df")
|
||||
plt.show()
|
||||
@@ -1,8 +1,8 @@
|
||||
import mlagents_envs
|
||||
from mlagents_envs.base_env import ActionTuple
|
||||
from mlagents_envs.environment import UnityEnvironment
|
||||
|
||||
import numpy as np
|
||||
from numpy import ndarray
|
||||
|
||||
|
||||
class makeEnv(object):
|
||||
@@ -22,69 +22,71 @@ class makeEnv(object):
|
||||
self.BEHA_SPECS = self.env.behavior_specs
|
||||
self.BEHA_NAME = list(self.BEHA_SPECS)[0]
|
||||
self.SPEC = self.BEHA_SPECS[self.BEHA_NAME]
|
||||
self.OBSERVATION_SPECS = self.SPEC.observation_specs[
|
||||
0
|
||||
] # observation spec
|
||||
self.OBSERVATION_SPECS = self.SPEC.observation_specs[0] # observation spec
|
||||
self.ACTION_SPEC = self.SPEC.action_spec # action specs
|
||||
|
||||
self.DISCRETE_SIZE = self.ACTION_SPEC.discrete_size # 連続的な動作のSize
|
||||
self.DISCRETE_SHAPE = list(self.ACTION_SPEC.discrete_branches)
|
||||
self.CONTINUOUS_SIZE = self.ACTION_SPEC.continuous_size # 離散的な動作のSize
|
||||
self.STATE_SIZE = (
|
||||
self.OBSERVATION_SPECS.shape[0] - self.LOAD_DIR_SIZE_IN_STATE
|
||||
) # 環境観測データ数
|
||||
self.STATE_SIZE = self.OBSERVATION_SPECS.shape[0] - self.LOAD_DIR_SIZE_IN_STATE # 環境観測データ数
|
||||
print("√√√√√Enviroment Initialized Success√√√√√")
|
||||
|
||||
def step(
|
||||
self,
|
||||
discreteActions=None,
|
||||
continuousActions=None,
|
||||
behaviorName=None,
|
||||
trackedAgent=None,
|
||||
actions: list,
|
||||
behaviorName: ndarray = None,
|
||||
trackedAgent: ndarray = None,
|
||||
):
|
||||
"""change ations list to ActionTuple then send it to enviroment
|
||||
|
||||
Args:
|
||||
actions (list): PPO chooseAction output action list
|
||||
behaviorName (ndarray, optional): behaviorName. Defaults to None.
|
||||
trackedAgent (ndarray, optional): trackedAgentID. Defaults to None.
|
||||
|
||||
Returns:
|
||||
ndarray: nextState, reward, done, loadDir, saveNow
|
||||
"""
|
||||
# take action to enviroment
|
||||
# return mextState,reward,done
|
||||
if self.DISCRETE_SIZE == 0:
|
||||
# create empty discrete action
|
||||
discreteActions = np.asarray([[0]])
|
||||
else:
|
||||
# create discrete action from actions list
|
||||
discreteActions = np.asanyarray([actions[0:self.DISCRETE_SIZE]])
|
||||
if self.CONTINUOUS_SIZE == 0:
|
||||
# create empty continuous action
|
||||
continuousActions = np.asanyarray([[0.0]])
|
||||
else:
|
||||
# create continuous actions from actions list
|
||||
continuousActions = np.asanyarray(actions[self.DISCRETE_SIZE:])
|
||||
|
||||
# check if arg is include None or IS None
|
||||
try:
|
||||
isDisNone = discreteActions.any() is None
|
||||
if discreteActions.all() is None:
|
||||
print("step() Error!:discreteActions include None")
|
||||
except:
|
||||
isDisNone = True
|
||||
try:
|
||||
isConNone = continuousActions.any() is None
|
||||
if continuousActions.all() is None:
|
||||
print("step() Error!:continuousActions include None")
|
||||
except:
|
||||
isConNone = True
|
||||
|
||||
if isDisNone:
|
||||
# if discreteActions is enpty just give nothing[[0]] to Enviroment
|
||||
discreteActions = np.array([[0]], dtype=np.int)
|
||||
if isConNone:
|
||||
# if continuousActions is enpty just give nothing[[0]] to Enviroment
|
||||
continuousActions = np.array([[0]], dtype=np.float)
|
||||
if behaviorName is None:
|
||||
behaviorName = self.BEHA_NAME
|
||||
if trackedAgent is None:
|
||||
trackedAgent = self.TRACKED_AGENT
|
||||
|
||||
# create actionTuple
|
||||
thisActionTuple = ActionTuple(
|
||||
continuous=continuousActions, discrete=discreteActions
|
||||
)
|
||||
thisActionTuple = ActionTuple(continuous=continuousActions, discrete=discreteActions)
|
||||
# take action to env
|
||||
self.env.set_actions(
|
||||
behavior_name=behaviorName, action=thisActionTuple
|
||||
)
|
||||
self.env.set_actions(behavior_name=behaviorName, action=thisActionTuple)
|
||||
self.env.step()
|
||||
# get nextState & reward & done after this action
|
||||
nextState, reward, done, loadDir, saveNow = self.getSteps(
|
||||
behaviorName, trackedAgent
|
||||
)
|
||||
nextState, reward, done, loadDir, saveNow = self.getSteps(behaviorName, trackedAgent)
|
||||
return nextState, reward, done, loadDir, saveNow
|
||||
|
||||
def getSteps(self, behaviorName=None, trackedAgent=None):
|
||||
"""get enviroment now observations.
|
||||
Include State, Reward, Done, LoadDir, SaveNow
|
||||
|
||||
Args:
|
||||
behaviorName (_type_, optional): behaviorName. Defaults to None.
|
||||
trackedAgent (_type_, optional): trackedAgent. Defaults to None.
|
||||
|
||||
Returns:
|
||||
ndarray: nextState, reward, done, loadDir, saveNow
|
||||
"""
|
||||
# get nextState & reward & done
|
||||
if behaviorName is None:
|
||||
behaviorName = self.BEHA_NAME
|
||||
@@ -94,25 +96,17 @@ class makeEnv(object):
|
||||
if trackedAgent is None:
|
||||
trackedAgent = self.TRACKED_AGENT
|
||||
|
||||
if (
|
||||
trackedAgent in decisionSteps
|
||||
): # ゲーム終了していない場合、環境状態がdecision_stepsに保存される
|
||||
if trackedAgent in decisionSteps: # ゲーム終了していない場合、環境状態がdecision_stepsに保存される
|
||||
nextState = decisionSteps[trackedAgent].obs[0]
|
||||
nextState = np.reshape(
|
||||
nextState, [1, self.STATE_SIZE + self.LOAD_DIR_SIZE_IN_STATE]
|
||||
)
|
||||
nextState = np.reshape(nextState, [1, self.STATE_SIZE + self.LOAD_DIR_SIZE_IN_STATE])
|
||||
saveNow = nextState[0][-1]
|
||||
loadDir = nextState[0][-3:-1]
|
||||
nextState = nextState[0][:-3]
|
||||
reward = decisionSteps[trackedAgent].reward
|
||||
done = False
|
||||
if (
|
||||
trackedAgent in terminalSteps
|
||||
): # ゲーム終了した場合、環境状態がterminal_stepsに保存される
|
||||
if trackedAgent in terminalSteps: # ゲーム終了した場合、環境状態がterminal_stepsに保存される
|
||||
nextState = terminalSteps[trackedAgent].obs[0]
|
||||
nextState = np.reshape(
|
||||
nextState, [1, self.STATE_SIZE + self.LOAD_DIR_SIZE_IN_STATE]
|
||||
)
|
||||
nextState = np.reshape(nextState, [1, self.STATE_SIZE + self.LOAD_DIR_SIZE_IN_STATE])
|
||||
saveNow = nextState[0][-1]
|
||||
loadDir = nextState[0][-3:-1]
|
||||
nextState = nextState[0][:-3]
|
||||
@@ -121,9 +115,16 @@ class makeEnv(object):
|
||||
return nextState, reward, done, loadDir, saveNow
|
||||
|
||||
def reset(self):
|
||||
"""reset enviroment and get observations
|
||||
|
||||
Returns:
|
||||
ndarray: nextState, reward, done, loadDir, saveNow
|
||||
"""
|
||||
self.env.reset()
|
||||
nextState, reward, done, loadDir, saveNow = self.getSteps()
|
||||
return nextState, reward, done, loadDir, saveNow
|
||||
|
||||
def render(self):
|
||||
"""render enviroment
|
||||
"""
|
||||
self.env.render()
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
import numpy as np
|
||||
|
||||
class buffer(object):
|
||||
def __init__(self):
|
||||
self.states = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
print("√√√√√Buffer Initialized Success√√√√√")
|
||||
def clearBuffer(self):
|
||||
self.states = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
def getStates(self):
|
||||
return np.asarray(self.states)
|
||||
def getActions(self):
|
||||
return np.asarray(self.actions)
|
||||
def getRewards(self):
|
||||
return np.asarray(self.rewards)
|
||||
|
||||
def saveState(self,state):
|
||||
self.states.append(state)
|
||||
def saveAction(self,action):
|
||||
self.actions.append(action)
|
||||
def saveReward(self,reward):
|
||||
self.rewards.append(reward)
|
||||
def saveBuffers(self,state,action,reward):
|
||||
self.states.append(state)
|
||||
self.actions.append(action)
|
||||
self.rewards.append(reward)
|
||||
@@ -1,356 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import aimBotEnv\n",
|
||||
"import PPO\n",
|
||||
"import buffer\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"import time\n",
|
||||
"import datetime\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Attempts to allocate only the GPU memory needed for allocation\n",
|
||||
"physical_devices = tf.config.list_physical_devices('GPU')\n",
|
||||
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Env\n",
|
||||
"ENV_PATH = \"./Build-CloseEnemyCut/Aimbot-PPO\"\n",
|
||||
"WORKER_ID = 1\n",
|
||||
"BASE_PORT = 200\n",
|
||||
"\n",
|
||||
"MAX_EP = 1000\n",
|
||||
"EP_LENGTH = 100000\n",
|
||||
"GAMMA = 0.99 # discount future reward (UP?)\n",
|
||||
"EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]\n",
|
||||
"ACTOR_LR = 1e-5 # LR\n",
|
||||
"CRITIC_LR = 2e-5 # LR\n",
|
||||
"BATCH = 256 # learning step\n",
|
||||
"ACTOR_EPOCH = 15 # epoch\n",
|
||||
"CRITIC_EPOCH = 15 # epoch\n",
|
||||
"ENTROPY_WHEIGHT = 0.001 # sigma's entropy in Actor loss\n",
|
||||
"ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"TRAIN = True\n",
|
||||
"SAVE_DIR = \"PPO-Model/\" + datetime.datetime.now().strftime(\"%m%d%H%M\") + \"/\"\n",
|
||||
"LOAD_DIR = None\n",
|
||||
"\n",
|
||||
"CTN_ACTION_RANGE = 10\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"√√√√√Enviroment Initialized Success√√√√√\n",
|
||||
"√√√√√Buffer Initialized Success√√√√√\n",
|
||||
"No loadDir specified,Create a New Model\n",
|
||||
"CONTINUOUS_SIZE 1\n",
|
||||
"DISCRETE_SIZE 5\n",
|
||||
"STATE_SIZE 30\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# initialize enviroment & buffer class\n",
|
||||
"env = aimBotEnv.makeEnv(\n",
|
||||
" envPath=ENV_PATH, workerID=WORKER_ID, basePort=BASE_PORT\n",
|
||||
")\n",
|
||||
"epBuffer = buffer.buffer()\n",
|
||||
"\n",
|
||||
"STATE_SIZE = env.STATE_SIZE\n",
|
||||
"CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n",
|
||||
"DISCRETE_SIZE = env.DISCRETE_SIZE\n",
|
||||
"_, _, _, loadDir, _ = env.getSteps()\n",
|
||||
"\n",
|
||||
"# check load model or not\n",
|
||||
"if np.any(loadDir == 0):\n",
|
||||
" # create a new model\n",
|
||||
" print(\"No loadDir specified,Create a New Model\")\n",
|
||||
" LOAD_DIR = None\n",
|
||||
"else:\n",
|
||||
" # load model\n",
|
||||
" loadDirDateSTR = str(int(loadDir[0]))\n",
|
||||
" loadDirTimeSTR = str(int(loadDir[1]))\n",
|
||||
" if len(loadDirDateSTR) != 8:\n",
|
||||
" # fill lost 0 while converse float to string\n",
|
||||
" for _ in range(8 - len(loadDirDateSTR)):\n",
|
||||
" loadDirDateSTR = \"0\" + loadDirDateSTR\n",
|
||||
" if len(loadDirTimeSTR) != 6:\n",
|
||||
" # fill lost 0 while converse float to string\n",
|
||||
" for _ in range(6 - len(loadDirTimeSTR)):\n",
|
||||
" loadDirTimeSTR = \"0\" + loadDirTimeSTR\n",
|
||||
" LOAD_DIR = \"PPO-Model/\" + loadDirDateSTR + \"/\" + loadDirTimeSTR\n",
|
||||
" print(\"Load Model:\")\n",
|
||||
" print(LOAD_DIR)\n",
|
||||
"\n",
|
||||
"print(\"CONTINUOUS_SIZE\", CONTINUOUS_SIZE)\n",
|
||||
"print(\"DISCRETE_SIZE\", DISCRETE_SIZE)\n",
|
||||
"print(\"STATE_SIZE\", STATE_SIZE)\n",
|
||||
"\n",
|
||||
"disActShape = [3, 3, 2]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def actToKey(disAct1,disAct2,disAct3,conAct):\n",
|
||||
" kW = 0\n",
|
||||
" kS = 0\n",
|
||||
" kA = 0\n",
|
||||
" kD = 0\n",
|
||||
" mouseShoot = 0\n",
|
||||
" if disAct1 == 0:\n",
|
||||
" kW = 0\n",
|
||||
" kS = 1\n",
|
||||
" elif disAct1 == 1:\n",
|
||||
" kW = 0\n",
|
||||
" kS = 0\n",
|
||||
" elif disAct1 == 2:\n",
|
||||
" kW = 1\n",
|
||||
" kS = 0\n",
|
||||
" if disAct2 == 0:\n",
|
||||
" kA = 0\n",
|
||||
" kD = 1\n",
|
||||
" elif disAct2 == 1:\n",
|
||||
" kA = 0\n",
|
||||
" kD = 0\n",
|
||||
" elif disAct2 == 2:\n",
|
||||
" kA = 1\n",
|
||||
" kD = 0\n",
|
||||
" mouseShoot = disAct3\n",
|
||||
" return kW,kS,kA,kD,mouseShoot,conAct"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"EP 0 START\n",
|
||||
"√√√√√Buffer Initialized Success√√√√√\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.\n",
|
||||
" return _methods._mean(a, axis=axis, dtype=dtype,\n",
|
||||
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
|
||||
" ret = ret.dtype.type(ret / rcount)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"A_Loss: 0.4477495511372884 C_Loss: 3.155759557088216\n",
|
||||
"A_Loss: 0.14549287557601928 C_Loss: 0.5123071213563283\n",
|
||||
"A_Loss: 0.055241942902406055 C_Loss: 0.13002794484297434\n",
|
||||
"A_Loss: 0.057325509190559384 C_Loss: 0.11068039039770762\n",
|
||||
"A_Loss: 0.04376962607105573 C_Loss: 0.03923700377345085\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"bestScore = 200.0\n",
|
||||
"stopTrainCounter = 0\n",
|
||||
"\n",
|
||||
"totalRewardHis = []\n",
|
||||
"totalActorLossHis = []\n",
|
||||
"totalCriticLossHis = []\n",
|
||||
"epHis = []\n",
|
||||
"maxTotalReward = -99999999999\n",
|
||||
"\n",
|
||||
"for ep in range(MAX_EP):\n",
|
||||
" print(\"EP \", ep, \" START\")\n",
|
||||
" # first time run game\n",
|
||||
" s, _, _, _, _ = env.reset()\n",
|
||||
" if ep == 0:\n",
|
||||
" epBuffer = buffer.buffer()\n",
|
||||
" s = s.reshape([STATE_SIZE])\n",
|
||||
" agent = PPO.PPO(\n",
|
||||
" stateSize=STATE_SIZE,\n",
|
||||
" disActShape=disActShape,\n",
|
||||
" conActSize=1,\n",
|
||||
" conActRange=CTN_ACTION_RANGE,\n",
|
||||
" criticLR=CRITIC_LR,\n",
|
||||
" actorLR=ACTOR_LR,\n",
|
||||
" gamma=GAMMA,\n",
|
||||
" epsilon=EPSILON,\n",
|
||||
" entropyWeight=ENTROPY_WHEIGHT,\n",
|
||||
" saveDir=SAVE_DIR,\n",
|
||||
" loadModelDir=LOAD_DIR,\n",
|
||||
" )\n",
|
||||
" step = 0\n",
|
||||
" done = False\n",
|
||||
" stopTrainCounter -= 1\n",
|
||||
" epHis.append(ep)\n",
|
||||
"\n",
|
||||
" # reset total reward\n",
|
||||
" epTotalReward = 0\n",
|
||||
"\n",
|
||||
" # Recorder list\n",
|
||||
" epStepHis = []\n",
|
||||
" epRewardHis = []\n",
|
||||
" epActorLossHis = []\n",
|
||||
" epCriticLossHis = []\n",
|
||||
"\n",
|
||||
" # save weight immediately?\n",
|
||||
" saveNow = 0\n",
|
||||
"\n",
|
||||
" while not done:\n",
|
||||
" step += 1\n",
|
||||
" if (\n",
|
||||
" step % ACTION_INTERVAL == 0\n",
|
||||
" ): # take action every ACTION_INTERVAL steps\n",
|
||||
" epStepHis.append(step)\n",
|
||||
" (\n",
|
||||
" disAct1,\n",
|
||||
" disAct2,\n",
|
||||
" disAct3,\n",
|
||||
" conAct,\n",
|
||||
" predictResult,\n",
|
||||
" ) = agent.chooseAction(s)\n",
|
||||
" kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(\n",
|
||||
" disAct1, disAct2, disAct3, conAct\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" nextState, thisReward, done, _, saveNow = env.step(\n",
|
||||
" discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),\n",
|
||||
" continuousActions=np.array([[mouseMove]]),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" epTotalReward += thisReward\n",
|
||||
" epBuffer.saveBuffers(\n",
|
||||
" s, [disAct1, disAct2, disAct3, conAct], thisReward\n",
|
||||
" )\n",
|
||||
" else:\n",
|
||||
" disActs = np.array([[0, 0, 0, 0, 0]])\n",
|
||||
" conActs = np.array([[0]])\n",
|
||||
"\n",
|
||||
" nextState, thisReward, done, _, saveNow = env.step(\n",
|
||||
" discreteActions=disActs, continuousActions=conActs\n",
|
||||
" )\n",
|
||||
" epTotalReward += thisReward\n",
|
||||
" nextState = nextState.reshape([STATE_SIZE])\n",
|
||||
" s = nextState\n",
|
||||
"\n",
|
||||
" if done:\n",
|
||||
" print(\"EP OVER!\")\n",
|
||||
" if saveNow != 0:\n",
|
||||
" print(\"SAVENOW!\")\n",
|
||||
" saveNow = 0\n",
|
||||
" agent.saveWeights()\n",
|
||||
" # update PPO after Batch step or GameOver\n",
|
||||
" if (step + 1) % BATCH == 0 or done:\n",
|
||||
" bs = epBuffer.getStates()\n",
|
||||
" ba = epBuffer.getActions()\n",
|
||||
" br = epBuffer.getRewards()\n",
|
||||
" epBuffer.clearBuffer()\n",
|
||||
" if TRAIN:\n",
|
||||
" epActorLoss, epCriticLoss = agent.trainCritcActor(\n",
|
||||
" bs, ba, br, s, CRITIC_EPOCH, ACTOR_EPOCH\n",
|
||||
" )\n",
|
||||
" epActorLossHis.append(epActorLoss)\n",
|
||||
" epCriticLossHis.append(epCriticLoss)\n",
|
||||
" # update History Recorder\n",
|
||||
" totalActorLossHis.append(np.mean(epActorLossHis))\n",
|
||||
" totalCriticLossHis.append(np.mean(epCriticLossHis))\n",
|
||||
" totalRewardHis.append(epTotalReward)\n",
|
||||
"\n",
|
||||
" if epTotalReward > maxTotalReward and epTotalReward != 0:\n",
|
||||
" maxTotalReward = epTotalReward\n",
|
||||
" agent.saveWeights(epTotalReward)\n",
|
||||
" print(\"New Record! Save NN\", epTotalReward)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"aaa = 0\n",
|
||||
"aaa = 1\n",
|
||||
"aaa = 2\n",
|
||||
"aaa = 3\n",
|
||||
"aaa = 4\n",
|
||||
"aaa = 5\n",
|
||||
"aaa = 6\n",
|
||||
"aaa = 7\n",
|
||||
"aaa = 8\n",
|
||||
"aaa = 9\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"aaa = 0\n",
|
||||
"while aaa<10:\n",
|
||||
" print(\"aaa = \",aaa)\n",
|
||||
" aaa+=1"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.7 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.7"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import aimBotEnv
|
||||
import PPO
|
||||
|
||||
ENV_PATH = './Build/Aimbot-PPO'
|
||||
WORKER_ID = 100
|
||||
|
||||
MAX_EP = 1000
|
||||
EP_LENGTH = 400
|
||||
GAMMA = 0.99 # discount future reward (UP?)
|
||||
EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]
|
||||
ACTOR_LR = 1e-5 # LR
|
||||
CRITIC_LR = 2e-5 # LR
|
||||
BATCH = 32 # learning step
|
||||
ACTOR_EPOCH = 10 # epoch
|
||||
CRITIC_EPOCH = 10 # epoch
|
||||
ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss
|
||||
ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps
|
||||
TRAIN = True
|
||||
|
||||
env = aimBotEnv.makeEnv(envPath = ENV_PATH,workerID = WORKER_ID)
|
||||
STATE_SIZE = env.STATE_SIZE
|
||||
CONTINUOUS_SIZE = env.CONTINUOUS_SIZE
|
||||
DISCRETE_SIZE = env.DISCRETE_SIZE
|
||||
|
||||
CTN_ACTION_RANGE = 2
|
||||
@@ -29,7 +29,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -73,7 +73,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -118,7 +118,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -182,7 +182,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -205,7 +205,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -240,16 +240,16 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"True"
|
||||
"array([[0.]])"
|
||||
]
|
||||
},
|
||||
"execution_count": 20,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -259,28 +259,119 @@
|
||||
"\n",
|
||||
"a = np.array([10, 20, 30, 0])\n",
|
||||
"\n",
|
||||
"np.any(a == 0)\n"
|
||||
"np.asarray([[0.]])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"4"
|
||||
"1.5"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"asd = \"adsf\"\n",
|
||||
"len(asd)"
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"asd = [1,2,3,np.array([0.5]),np.array([0.5])]\n",
|
||||
"\n",
|
||||
"asd[3:]\n",
|
||||
"len(asd)\n",
|
||||
"\n",
|
||||
"np.mean([1,2])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0.0\n",
|
||||
"0.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import time\n",
|
||||
"import pyautogui as pag\n",
|
||||
"\n",
|
||||
"from pynput.mouse import Button, Controller\n",
|
||||
"\n",
|
||||
"w = pag.size().width\n",
|
||||
"h = pag.size().height\n",
|
||||
"mouse = Controller()\n",
|
||||
"\n",
|
||||
"nowt = time.time()\n",
|
||||
"\n",
|
||||
"middletime = time.time() - nowt\n",
|
||||
"print(middletime)\n",
|
||||
"# print(nowPos-(w/2))\n",
|
||||
"\n",
|
||||
"print(time.time() - middletime - nowt)\n",
|
||||
"while True:\n",
|
||||
" x,_ = mouse.position\n",
|
||||
" #print(mouse.press)\n",
|
||||
" #print(mouse.position)\n",
|
||||
" \n",
|
||||
" mouse.position = (w / 2, h / 2)\n",
|
||||
" time.sleep(1/60)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import time\n",
|
||||
"import pyautogui as pag\n",
|
||||
"\n",
|
||||
"import mouse\n",
|
||||
"\n",
|
||||
"w = pag.size().width\n",
|
||||
"h = pag.size().height\n",
|
||||
"\n",
|
||||
"nowt = time.time()\n",
|
||||
"\n",
|
||||
"middletime = time.time() - nowt\n",
|
||||
"print(middletime)\n",
|
||||
"# print(nowPos-(w/2))\n",
|
||||
"\n",
|
||||
"print(time.time() - middletime - nowt)\n",
|
||||
"while True:\n",
|
||||
" x = mouse.get_position()\n",
|
||||
" print(x)\n",
|
||||
" #print(mouse.position)\n",
|
||||
" \n",
|
||||
" mouse.move(w / 2, h / 2)\n",
|
||||
" time.sleep(1/60)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import keyboard\n",
|
||||
"\n",
|
||||
"while True:\n",
|
||||
" if keyboard.is_pressed(\"w\"):\n",
|
||||
" print(\"w\")\n",
|
||||
" elif keyboard.is_pressed(\"s\"):\n",
|
||||
" print(\"s\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user