Aimbot Enviroment very first
Basic environment include Multi scene, Reward Change, Visible chart, etc....
This commit is contained in:
@@ -0,0 +1,462 @@
|
||||
import tensorflow as tf
|
||||
import tensorflow_probability as tfp
|
||||
import numpy as np
|
||||
import math
|
||||
import copy
|
||||
import datetime
|
||||
import os
|
||||
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
from tensorflow.keras import optimizers
|
||||
from keras_radam import RAdam
|
||||
|
||||
|
||||
class PPO(object):
|
||||
"""Create PPO Agent
|
||||
"""
|
||||
|
||||
def __init__(self, stateSize, disActShape, conActSize, conActRange, criticLR, actorLR, gamma, epsilon, entropyWeight, saveDir, loadModelDir):
|
||||
|
||||
# check disActShape is correct(greater than 1)
|
||||
try:
|
||||
if np.any(np.array(disActShape)<=1):
|
||||
raise ValueError("disActShape error,disActShape should greater than 1 but get",disActShape)
|
||||
except ValueError as e:
|
||||
raise
|
||||
|
||||
self.stateSize = stateSize
|
||||
# self.actionSize = actionSize
|
||||
self.disActShape = disActShape # shape of discrete action output. like [3,3,2]
|
||||
self.disActSize = len(disActShape)
|
||||
self.conActSize = conActSize
|
||||
self.conActRange = conActRange
|
||||
self.criticLR = criticLR
|
||||
self.actorLR = actorLR
|
||||
self.GAMMA = gamma
|
||||
self.EPSILON = epsilon
|
||||
self.saveDir = saveDir
|
||||
self.entropyWeight = entropyWeight
|
||||
|
||||
self.disOutputSize = sum(disActShape)
|
||||
self.conOutputSize = conActSize * 2
|
||||
|
||||
if loadModelDir == None:
|
||||
# critc NN
|
||||
self.critic = self.buildCriticNet(self.stateSize, 1, compileModel = True)
|
||||
# actor NN
|
||||
self.actor = self.buildActorNet(self.stateSize, self.conActRange, compileModel = True)
|
||||
else:
|
||||
# critc NN
|
||||
self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
|
||||
# actor NN
|
||||
self.actor = self.buildActorNet(self.stateSize, self.conActRange, compileModel=True)
|
||||
# load weight to Critic&Actor NN
|
||||
self.loadWeightToModels(loadModelDir)
|
||||
|
||||
|
||||
# Build Net
|
||||
def buildActorNet(self, inputSize, continuousActionRange,compileModel):
|
||||
"""build Actor Nueral Net and compile.Output:[disAct1,disAct2,disAct3,mu,sigma]
|
||||
|
||||
Args:
|
||||
inputSize (int): InputLayer Nueral size.
|
||||
continuousActionRange (foat): continuous Action's max Range.
|
||||
|
||||
Returns:
|
||||
keras.Model: return Actor NN
|
||||
"""
|
||||
stateInput = layers.Input(shape=(inputSize,), name='stateInput')
|
||||
dense0 = layers.Dense(500, activation='relu',name='dense0',)(stateInput)
|
||||
dense1 = layers.Dense(200, activation='relu',name='dense1',)(dense0)
|
||||
dense2 = layers.Dense(100, activation='relu', name='dense2')(dense1)
|
||||
|
||||
disAct1 = layers.Dense(3, activation='softmax',name='WSAction')(dense2) # WS
|
||||
disAct2 = layers.Dense(3, activation='softmax',name='ADAction')(dense2) # AD
|
||||
disAct3 = layers.Dense(2, activation='softmax',name='ShootAction')(dense2) # Mouse shoot
|
||||
mu = continuousActionRange * layers.Dense(1, activation='tanh', name='muOut')(dense2) # mu，既正态分布mean
|
||||
sigma = 1e-8 + layers.Dense(1, activation='softplus',name='sigmaOut')(dense2) # sigma，既正态分布
|
||||
# musig = layers.concatenate([mu,sigma],name = 'musig')
|
||||
totalOut = layers.concatenate(
|
||||
[disAct1, disAct2, disAct3, mu, sigma], name='totalOut') # package
|
||||
|
||||
model = keras.Model(inputs=stateInput, outputs=totalOut)
|
||||
#actorOPT = optimizers.Adam(learning_rate = self.actorLR)
|
||||
if compileModel:
|
||||
actorOPT = RAdam(self.actorLR)
|
||||
model.compile(optimizer=actorOPT, loss=self.aLoss())
|
||||
return model
|
||||
|
||||
def buildCriticNet(self, inputSize, outputSize,compileModel):
|
||||
"""build Critic Nueral Net and compile.Output:[Q]
|
||||
|
||||
Args:
|
||||
inputSize (int): InputLayer Neural Size
|
||||
outputSize (float): Q size
|
||||
|
||||
Returns:
|
||||
keras.Model: return Critic NN
|
||||
"""
|
||||
stateInput = keras.Input(shape=(inputSize,))
|
||||
dense0 = layers.Dense(500, activation='relu',
|
||||
name='dense0',)(stateInput)
|
||||
dense1 = layers.Dense(200, activation='relu')(dense0)
|
||||
dense2 = layers.Dense(100, activation='relu')(dense1)
|
||||
output = layers.Dense(outputSize)(dense2)
|
||||
model = keras.Model(inputs=stateInput, outputs=output)
|
||||
if compileModel:
|
||||
criticOPT = optimizers.Adam(learning_rate=self.criticLR)
|
||||
model.compile(optimizer=criticOPT, loss=self.cLoss())
|
||||
return model
|
||||
|
||||
# loss Function
|
||||
def cLoss(self):
|
||||
"""Critic Loss function
|
||||
"""
|
||||
def loss(y_true, y_pred):
|
||||
# y_true: discountedR
|
||||
# y_pred: critcV = model.predict(states)
|
||||
|
||||
advantage = y_true - y_pred # TD error
|
||||
loss = tf.reduce_mean(tf.square(advantage))
|
||||
return loss
|
||||
return loss
|
||||
|
||||
def aLoss(self):
|
||||
def getDiscreteALoss(nowProbs,oldProbs,advantage):
|
||||
"""get Discrete Action Loss
|
||||
|
||||
Args:
|
||||
nowProbs (tf.constant): (length,actionSize)
|
||||
oldProbs (tf.constant): (length,actionSize)
|
||||
advantage (tf.constant): (length,)
|
||||
|
||||
Returns:
|
||||
tf.constant: (length,)
|
||||
"""
|
||||
entropy = tf.reduce_mean(tf.math.multiply(nowProbs,tf.math.log(nowProbs+1e-6)))
|
||||
ratio = tf.math.divide(nowProbs,oldProbs+1e-6)
|
||||
value = tf.math.multiply(ratio,tf.expand_dims(advantage,axis = 1))
|
||||
clipRatio = tf.clip_by_value(ratio,1. - self.EPSILON,1.+self.EPSILON)
|
||||
clipValue = tf.math.multiply(clipRatio,tf.expand_dims(advantage,axis = 1))
|
||||
loss = -tf.reduce_mean(tf.math.minimum(value,clipValue)) + self.entropyWeight * entropy
|
||||
return loss
|
||||
|
||||
def getContinuousALoss(musig,actions,oldProbs,advantage):
|
||||
"""get Continuous Action Loss
|
||||
|
||||
Args:
|
||||
musig (tf.constant): (length,2)
|
||||
actions (tf.constant): (length,)
|
||||
oldProbs (tf.constant): (length,)
|
||||
advantage (tf.constant): (length,)
|
||||
|
||||
Returns:
|
||||
tf.constant: (length,)
|
||||
"""
|
||||
mu = musig[:,0]
|
||||
sigma = musig[:,1]
|
||||
dist = tfp.distributions.Normal(mu,sigma)
|
||||
|
||||
nowProbs = dist.prob(actions)
|
||||
ratio = tf.math.divide(nowProbs,oldProbs+1e-6)
|
||||
entropy = tf.reduce_mean(dist.entropy())
|
||||
|
||||
value = tf.math.multiply(ratio,tf.expand_dims(advantage,axis = 1))
|
||||
clipValue = tf.clip_by_value(ratio,1. - self.EPSILON,1.+self.EPSILON) * advantage
|
||||
loss = -tf.reduce_mean(tf.math.minimum(value,clipValue)) + self.entropyWeight * entropy
|
||||
return loss
|
||||
|
||||
def loss(y_true, y_pred):
|
||||
# y_true: [[disAct1, disAct2, disAct3, mu, sigma]]
|
||||
# y_pred: muSigma = self.actor(state) =
|
||||
# [[disAct1, disAct2, disAct3, mu, sigma]]
|
||||
oldDisProbs = y_true[:,0:self.disOutputSize]
|
||||
oldConMusigs = y_true[:,self.disOutputSize:self.disOutputSize+self.conActSize]
|
||||
conActions = y_true[:,self.disOutputSize+self.conActSize:self.disOutputSize+(self.conActSize*2)]
|
||||
advantage = y_true[:,-1]
|
||||
|
||||
nowDisProbs = y_pred[:,0:self.disOutputSize] # [disAct1, disAct2, disAct3]
|
||||
nowConMusigs = y_pred[:,self.disOutputSize:] #[musig1,musig2]
|
||||
|
||||
totalALoss = tf.constant([0.])
|
||||
totalActionNum = 0
|
||||
|
||||
# for nowProb,oldProb in zip(tf.transpose(nowDisProbs,perm=[1,0,2]),tf.transpose(oldDisProbs,perm=[1,0,2])):
|
||||
lastDisActShape = 0
|
||||
for shape in self.disActShape:
|
||||
thisNowDisProbs = nowDisProbs[:,lastDisActShape:lastDisActShape+shape]
|
||||
thisOldDisProbs = oldDisProbs[:,lastDisActShape:lastDisActShape+shape]
|
||||
discreteALoss = getDiscreteALoss(thisNowDisProbs,thisOldDisProbs,advantage)
|
||||
lastDisActShape += shape
|
||||
totalALoss += discreteALoss
|
||||
totalActionNum += 1
|
||||
# for nowConMusig,conAction,oldPiProb in zip(tf.transpose(nowConMusigs,perm=[1,0,2]),conActions,oldPiProbs):
|
||||
lastConAct = 0
|
||||
for act in range(self.conActSize):
|
||||
thisNowConMusig = nowConMusigs[:,lastConAct:lastConAct+((act+1)*2)]
|
||||
thisOldConMusig = oldConMusigs[:,lastConAct:lastConAct+((act+1)*2)]
|
||||
thisConAction = conActions[:,act]
|
||||
continuousAloss = getContinuousALoss(thisNowConMusig,thisConAction,thisOldConMusig,advantage)
|
||||
totalALoss += continuousAloss
|
||||
totalActionNum += 1
|
||||
|
||||
loss = tf.divide(totalALoss,totalActionNum)
|
||||
return loss
|
||||
return loss
|
||||
|
||||
# get Action&V
|
||||
def chooseAction(self, state):
|
||||
"""Agent choose action to take
|
||||
|
||||
Args:
|
||||
state (np.array): enviroment state
|
||||
|
||||
Returns:
|
||||
np.array:
|
||||
disAct1,
|
||||
discreteAction1
|
||||
disAct2,
|
||||
discreteAction2
|
||||
disAct3,
|
||||
discreteAction3
|
||||
conAction,
|
||||
continuousAction
|
||||
predictResult,
|
||||
actor NN predict Result output
|
||||
"""
|
||||
# let actor choose action,use the normal distribution
|
||||
# state = np.expand_dims(state,0)
|
||||
|
||||
# check state dimension is [1,statesize]
|
||||
if state.ndim!=2:
|
||||
state = state.reshape([1,self.stateSize])
|
||||
|
||||
predictResult = self.actor(state) # get predict result [[disAct1, disAct2, disAct3, musig]]
|
||||
predictResult = predictResult.numpy()
|
||||
disAct1Prob = predictResult[0][0:3]
|
||||
disAct2Prob = predictResult[0][3:6]
|
||||
disAct3Prob = predictResult[0][6:8]
|
||||
mu = predictResult[0][8]
|
||||
sigma = predictResult[0][9]
|
||||
if math.isnan(mu) or math.isnan(sigma):
|
||||
# check mu or sigma is nan
|
||||
print("mu or sigma is nan")
|
||||
|
||||
disAct1 = np.argmax(disAct1Prob) # WS 0 or 1 or 2
|
||||
disAct2 = np.argmax(disAct2Prob) # AD 0 or 1 or 2
|
||||
disAct3 = np.argmax(disAct3Prob) # mouse shoot 0 or 1
|
||||
normDist = np.random.normal(loc=mu, scale=sigma) # normalDistribution
|
||||
conAction = np.clip(normDist, -self.conActRange,
|
||||
self.conActRange) # 在正态分布中随机get一个action
|
||||
return disAct1, disAct2, disAct3, conAction, predictResult
|
||||
|
||||
def getCriticV(self, state):
|
||||
"""get Critic predict V value
|
||||
|
||||
Args:
|
||||
state (np.array): Env state
|
||||
|
||||
Returns:
|
||||
tensor: retrun Critic predict result
|
||||
"""
|
||||
# if state.ndim < 2:
|
||||
# state = np.expand_dims(state,0)
|
||||
if state.ndim!=2:
|
||||
state = state.reshape([1,self.stateSize])
|
||||
return self.critic.predict(state)
|
||||
|
||||
def discountReward(self, nextState, rewards):
|
||||
"""Discount future rewards
|
||||
|
||||
Args:
|
||||
nextState (np.array): next Env state
|
||||
rewards (np.array): reward list of this episode
|
||||
|
||||
Returns:
|
||||
np.array: discounted rewards list,same shape as rewards that input
|
||||
"""
|
||||
# 降低未来的rewards
|
||||
nextV = self.getCriticV(nextState)
|
||||
discountedRewards = []
|
||||
for r in rewards[::-1]:
|
||||
nextV = r + self.GAMMA*nextV
|
||||
discountedRewards.append(nextV)
|
||||
discountedRewards.reverse() # \ESREVER/
|
||||
discountedRewards = np.squeeze(discountedRewards)
|
||||
discountedRewards = np.expand_dims(discountedRewards, axis=1)
|
||||
#discountedRewards = np.array(discountedRewards)[:, np.newaxis]
|
||||
return discountedRewards
|
||||
|
||||
def conProb(self, mu, sig, x):
|
||||
"""calculate probability when x in Normal distribution(mu,sigma)
|
||||
|
||||
Args:
|
||||
mu (np,array): mu
|
||||
sig (np.array): sigma
|
||||
x (np.array): x
|
||||
|
||||
Returns:
|
||||
np.array: probabilities
|
||||
"""
|
||||
# 获取在正态分布mu,sig下当取x值时的概率
|
||||
# return shape : (length,1)
|
||||
mu = np.reshape(mu, (np.size(mu),))
|
||||
sig = np.reshape(sig, (np.size(sig),))
|
||||
x = np.reshape(x, (np.size(x),))
|
||||
|
||||
dist = tfp.distributions.Normal(mu, sig)
|
||||
prob = dist.prob(x)
|
||||
|
||||
prob = np.reshape(prob, (np.size(x), 1))
|
||||
#dist = 1./(tf.sqrt(2.*np.pi)*sig)
|
||||
#prob = dist*tf.exp(-tf.square(x-mu)/(2.*tf.square(sig)))
|
||||
return prob
|
||||
|
||||
def trainCritcActor(self, states, actions, rewards, nextState, criticEpochs, actorEpochs):
|
||||
# Train ActorNN and CriticNN
|
||||
# states: Buffer States
|
||||
# actions: Buffer Actions
|
||||
# rewards: Buffer Rewards,没有Discount处理
|
||||
# nextState: 下一个单独state
|
||||
# criticEpochs: just criticNN'Epochs
|
||||
# acotrEpochs: just acotrNN'Epochs
|
||||
discountedR = self.discountReward(nextState, rewards)
|
||||
|
||||
criticMeanLoss = self.trainCritic(states, discountedR, criticEpochs)
|
||||
actorMeanLoss = self.trainActor(
|
||||
states, actions, discountedR, actorEpochs)
|
||||
print("A_Loss:", actorMeanLoss, "C_Loss:", criticMeanLoss)
|
||||
return actorMeanLoss, criticMeanLoss
|
||||
|
||||
def trainCritic(self, states, discountedR, epochs):
|
||||
# Trian Critic
|
||||
# states: Buffer States
|
||||
# discountedR: Discounted Rewards
|
||||
# Epochs: just Epochs
|
||||
|
||||
# IDK why this should be list...It just work...
|
||||
# If discountR in np.array type it will throw 'Failed to find data adapter that can handle'
|
||||
# discountedR = discountedR.tolist()
|
||||
his = self.critic.fit(x=states, y=discountedR,
|
||||
epochs=epochs, verbose=0)
|
||||
return np.mean(his.history['loss'])
|
||||
|
||||
def trainActor(self, states, actions, discountedR, epochs):
|
||||
"""Actor NN trainning function
|
||||
|
||||
Args:
|
||||
states (np.array): Env states
|
||||
actions (np.array): action history
|
||||
discountedR (np.array): discountedR
|
||||
epochs (int): epochs,how many time NN learning
|
||||
|
||||
Returns:
|
||||
Average actor loss: this learning round's average actor loss
|
||||
"""
|
||||
# Trian Actor
|
||||
# states: Buffer States
|
||||
# actions: Buffer Actions
|
||||
# discountedR: Discounted Rewards
|
||||
# Epochs: just Epochs
|
||||
|
||||
states = np.asarray(states)
|
||||
actions = np.asarray(actions, dtype=np.float32)
|
||||
# predict with old Actor NN
|
||||
oldActorResult = self.actor.predict(states)
|
||||
|
||||
# assembly Actions history
|
||||
disActions = actions[:,0:self.disActSize]
|
||||
conActions = actions[:,self.disActSize:]
|
||||
# assembly predictResult as old Actor's Result
|
||||
oldDisProbs = oldActorResult[:,0:self.disOutputSize] # [disAct1, disAct2, disAct3]
|
||||
oldConMusigs = oldActorResult[:,self.disOutputSize:] # [musig1,musig2]
|
||||
oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
|
||||
|
||||
criticV = self.critic.predict(states)
|
||||
advantage = copy.deepcopy(discountedR - criticV)
|
||||
|
||||
# pack [oldDisProbs,oldPiProbs,conActions,advantage] as y_true
|
||||
y_true = np.hstack((oldDisProbs,oldPiProbs,conActions,advantage))
|
||||
|
||||
# train start
|
||||
if np.any(tf.math.is_nan(y_true)):
|
||||
print("y_true got nan")
|
||||
print("oldConMusigs",oldConMusigs)
|
||||
print("oldPiProbs",oldPiProbs)
|
||||
print("conActions",conActions)
|
||||
print("oldConMusigs",oldConMusigs)
|
||||
his = self.actor.fit(x=states, y=y_true, epochs=epochs, verbose=0)
|
||||
if np.any(tf.math.is_nan(his.history['loss'])):
|
||||
print("his.history['loss'] is nan!")
|
||||
print(his.history['loss'])
|
||||
return np.mean(his.history['loss'])
|
||||
|
||||
def saveWeights(self,score):
|
||||
"""save now NN's Weight. Use "models.save_weights" method.
|
||||
Save as "tf" format "ckpt" file.
|
||||
|
||||
Args:
|
||||
score (int): now score
|
||||
"""
|
||||
actor_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
|
||||
critic_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
|
||||
score_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
|
||||
self.actor.save_weights(actor_save_dir, save_format="tf")
|
||||
self.critic.save_weights(critic_save_dir, save_format="tf")
|
||||
# create an empty file named as score to recored score
|
||||
scorefile = open(score_dir,'w')
|
||||
scorefile.close()
|
||||
print("Model's Weights Saved")
|
||||
|
||||
def loadWeightToModels(self,loadDir):
|
||||
"""load NN Model. Use "models.load_weights()" method.
|
||||
Load "tf" format "ckpt" file.
|
||||
|
||||
Args:
|
||||
loadDir (string): Model dir
|
||||
"""
|
||||
actorDir = loadDir + "/actor/" + "actor.ckpt"
|
||||
criticDir = loadDir + "/critic/" + "critic.ckpt"
|
||||
self.actor.load_weights(actorDir)
|
||||
self.critic.load_weights(criticDir)
|
||||
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
print("++++++++++++Model Loaded++++++++++++")
|
||||
print(loadDir)
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
|
||||
def saveModel(self, score):
|
||||
"""save now NN Model. Use "model.save()" method.
|
||||
|
||||
Args:
|
||||
score (int): now score
|
||||
"""
|
||||
score = "_" + str(round(score))
|
||||
actor_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + \
|
||||
score+"/actor.h5"
|
||||
critic_save_dir = self.saveDir+datetime.datetime.now().strftime("%H%M%S") + \
|
||||
score+"/critic.h5"
|
||||
self.actor.save(actor_save_dir)
|
||||
self.critic.save(critic_save_dir)
|
||||
print("Model Saved")
|
||||
|
||||
def loadModel(self, loadDir):
|
||||
"""load NN Model. Use "models.load_model()" method.
|
||||
|
||||
Args:
|
||||
loadDir (string): Model dir
|
||||
|
||||
Returns:
|
||||
tf.keras.models: retuen compiled models.
|
||||
"""
|
||||
actorDir = loadDir+"/actor.h5"
|
||||
criticDir = loadDir+"/critic.h5"
|
||||
actor_net_loaded = tf.keras.models.load_model(actorDir)
|
||||
critic_net_loaded = tf.keras.models.load_model(criticDir)
|
||||
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
print("++++++++++++Model Loaded++++++++++++")
|
||||
print(loadDir)
|
||||
print("++++++++++++++++++++++++++++++++++++")
|
||||
return actor_net_loaded, critic_net_loaded
|
||||
@@ -0,0 +1,96 @@
|
||||
import mlagents_envs
|
||||
from mlagents_envs.base_env import ActionTuple
|
||||
from mlagents_envs.environment import UnityEnvironment
|
||||
|
||||
import numpy as np
|
||||
|
||||
class makeEnv(object):
|
||||
def __init__(self,envPath,workerID,basePort):
|
||||
self.env = UnityEnvironment(file_name=envPath,seed = 1,side_channels=[],worker_id = workerID,base_port=basePort)
|
||||
self.env.reset()
|
||||
|
||||
# get enviroment specs
|
||||
self.LOAD_DIR_SIZE_IN_STATE = 2
|
||||
self.TRACKED_AGENT = -1
|
||||
self.BEHA_SPECS = self.env.behavior_specs
|
||||
self.BEHA_NAME = list(self.BEHA_SPECS)[0]
|
||||
self.SPEC = self.BEHA_SPECS[self.BEHA_NAME]
|
||||
self.OBSERVATION_SPECS = self.SPEC.observation_specs[0] # observation spec
|
||||
self.ACTION_SPEC = self.SPEC.action_spec # action specs
|
||||
|
||||
self.DISCRETE_SIZE = self.ACTION_SPEC.discrete_size# 連続的な動作のSize
|
||||
self.CONTINUOUS_SIZE = self.ACTION_SPEC.continuous_size# 離散的な動作のSize
|
||||
self.STATE_SIZE = self.OBSERVATION_SPECS.shape[0] - self.LOAD_DIR_SIZE_IN_STATE# 環境観測データ数
|
||||
print("√√√√√Enviroment Initialized Success√√√√√")
|
||||
|
||||
def step(self,discreteActions = None,continuousActions = None,behaviorName = None,trackedAgent = None):
|
||||
# take action to enviroment
|
||||
# return mextState,reward,done
|
||||
|
||||
# check if arg is include None or IS None
|
||||
try:
|
||||
isDisNone = discreteActions.any() == None
|
||||
if discreteActions.all() == None:
|
||||
print("step() Error!:discreteActions include None")
|
||||
except:
|
||||
isDisNone = True
|
||||
try:
|
||||
isConNone = continuousActions.any() == None
|
||||
if continuousActions.all() == None:
|
||||
print("step() Error!:continuousActions include None")
|
||||
except:
|
||||
isConNone = True
|
||||
|
||||
if isDisNone:
|
||||
# if discreteActions is enpty just give nothing[[0]] to Enviroment
|
||||
discreteActions = np.array([[0]], dtype=np.int)
|
||||
if isConNone:
|
||||
# if continuousActions is enpty just give nothing[[0]] to Enviroment
|
||||
continuousActions = np.array([[0]], dtype=np.float)
|
||||
if behaviorName == None:
|
||||
behaviorName = self.BEHA_NAME
|
||||
if trackedAgent == None:
|
||||
trackedAgent = self.TRACKED_AGENT
|
||||
|
||||
#create actionTuple
|
||||
thisActionTuple = ActionTuple(continuous=continuousActions,discrete=discreteActions)
|
||||
# take action to env
|
||||
self.env.set_actions(behavior_name=behaviorName,action=thisActionTuple)
|
||||
self.env.step()
|
||||
# get nextState & reward & done after this action
|
||||
nextState,reward,done,loadDir = self.getSteps(behaviorName,trackedAgent)
|
||||
return nextState,reward,done,loadDir
|
||||
|
||||
def getSteps(self,behaviorName = None,trackedAgent = None):
|
||||
# get nextState & reward & done
|
||||
if behaviorName == None:
|
||||
behaviorName = self.BEHA_NAME
|
||||
decisionSteps,terminalSteps = self.env.get_steps(behaviorName)
|
||||
if self.TRACKED_AGENT == -1 and len(decisionSteps) >= 1:
|
||||
self.TRACKED_AGENT = decisionSteps.agent_id[0]
|
||||
if trackedAgent == None:
|
||||
trackedAgent = self.TRACKED_AGENT
|
||||
|
||||
if trackedAgent in decisionSteps: # ゲーム終了していない場合、環境状態がdecision_stepsに保存される
|
||||
nextState = decisionSteps[trackedAgent].obs[0]
|
||||
nextState = np.reshape(nextState,[1,self.STATE_SIZE+self.LOAD_DIR_SIZE_IN_STATE])
|
||||
loadDir = nextState[0][-2:]
|
||||
nextState = nextState[0][:-2]
|
||||
reward = decisionSteps[trackedAgent].reward
|
||||
done = False
|
||||
if trackedAgent in terminalSteps: # ゲーム終了した場合、環境状態がterminal_stepsに保存される
|
||||
nextState = terminalSteps[trackedAgent].obs[0]
|
||||
nextState = np.reshape(nextState,[1,self.STATE_SIZE+self.LOAD_DIR_SIZE_IN_STATE])
|
||||
loadDir = nextState[0][-2:]
|
||||
nextState = nextState[0][:-2]
|
||||
reward = terminalSteps[trackedAgent].reward
|
||||
done = True
|
||||
return nextState, reward, done, loadDir
|
||||
|
||||
def reset(self):
|
||||
self.env.reset()
|
||||
nextState,reward,done,loadDir = self.getSteps()
|
||||
return nextState,reward,done,loadDir
|
||||
|
||||
def render(self):
|
||||
self.env.render()
|
||||
@@ -0,0 +1,29 @@
|
||||
import numpy as np
|
||||
|
||||
class buffer(object):
|
||||
def __init__(self):
|
||||
self.states = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
print("√√√√√Buffer Initialized Success√√√√√")
|
||||
def clearBuffer(self):
|
||||
self.states = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
def getStates(self):
|
||||
return np.asarray(self.states)
|
||||
def getActions(self):
|
||||
return np.asarray(self.actions)
|
||||
def getRewards(self):
|
||||
return np.asarray(self.rewards)
|
||||
|
||||
def saveState(self,state):
|
||||
self.states.append(state)
|
||||
def saveAction(self,action):
|
||||
self.actions.append(action)
|
||||
def saveReward(self,reward):
|
||||
self.rewards.append(reward)
|
||||
def saveBuffers(self,state,action,reward):
|
||||
self.states.append(state)
|
||||
self.actions.append(action)
|
||||
self.rewards.append(reward)
|
||||
@@ -0,0 +1,158 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import env"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"31\n",
|
||||
"5\n",
|
||||
"3\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"包含None！\n",
|
||||
"[123 None]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"def qa(aa = None):\n",
|
||||
" try:\n",
|
||||
" isaanone = aa.any() == None\n",
|
||||
" if aa.all() == None:\n",
|
||||
" print(\"包含None！\")\n",
|
||||
" except:\n",
|
||||
" isaanone =True\n",
|
||||
" if isaanone:\n",
|
||||
" print('none')\n",
|
||||
" else:\n",
|
||||
" print(aa)\n",
|
||||
"\n",
|
||||
"qa(np.array([123,None]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[[[1 1 1 1 1]\n",
|
||||
" [1 2 1 1 1]]\n",
|
||||
"\n",
|
||||
" [[2 1 3 4 5]\n",
|
||||
" [2 2 3 4 5]]\n",
|
||||
"\n",
|
||||
" [[3 1 3 4 5]\n",
|
||||
" [3 2 3 4 5]]]\n",
|
||||
"-\n",
|
||||
"tf.Tensor(\n",
|
||||
"[[1 1 1 1 1]\n",
|
||||
" [2 1 3 4 5]\n",
|
||||
" [3 1 3 4 5]], shape=(3, 5), dtype=int32)\n",
|
||||
"-\n",
|
||||
"tf.Tensor(\n",
|
||||
"[[1 2 1 1 1]\n",
|
||||
" [2 2 3 4 5]\n",
|
||||
" [3 2 3 4 5]], shape=(3, 5), dtype=int32)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"aa = np.array([[[1,1,1,1,1],[1,2,1,1,1],[1,3,1,1,1]],\n",
|
||||
" [[2,1,3,4,5],[2,2,3,4,5],[2,3,3,4,5]],\n",
|
||||
" [[3,1,3,4,5],[3,2,3,4,5],[3,3,3,4,5]]])\n",
|
||||
"tt = tf.constant(aa)\n",
|
||||
"bb = np.array([6,3,6,3,2,3])\n",
|
||||
"\n",
|
||||
"print(aa[:,0:2])\n",
|
||||
"aa[:,2:]\n",
|
||||
"\n",
|
||||
"for asd in tf.transpose(aa[:,0:2],perm=[1,0,2]):\n",
|
||||
" print('-')\n",
|
||||
" print(asd)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 35,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<tf.Tensor: shape=(), dtype=int32, numpy=1>"
|
||||
]
|
||||
},
|
||||
"execution_count": 35,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"aa = tf.constant(0)\n",
|
||||
"bb = aa+1\n",
|
||||
"bb\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.7 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.7"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import aimBotEnv\n",
|
||||
"import PPO\n",
|
||||
"import buffer\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"import time\n",
|
||||
"import datetime"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Attempts to allocate only the GPU memory needed for allocation\n",
|
||||
"physical_devices = tf.config.list_physical_devices('GPU')\n",
|
||||
"tf.config.experimental.set_memory_growth(physical_devices[0], True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ENV_PATH = './Build-MultiScene-WithLoad/Aimbot-PPO'\n",
|
||||
"WORKER_ID = 1\n",
|
||||
"BASE_PORT = 200\n",
|
||||
"\n",
|
||||
"MAX_EP = 1000\n",
|
||||
"EP_LENGTH = 100000\n",
|
||||
"GAMMA = 0.99 # discount future reward (UP?)\n",
|
||||
"EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]\n",
|
||||
"ACTOR_LR = 1e-5 # LR\n",
|
||||
"CRITIC_LR = 2e-5 # LR\n",
|
||||
"BATCH = 512 # learning step\n",
|
||||
"ACTOR_EPOCH = 15 # epoch\n",
|
||||
"CRITIC_EPOCH = 15 # epoch\n",
|
||||
"ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss\n",
|
||||
"ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"TRAIN = True\n",
|
||||
"SAVE_DIR = \"PPO-Model/\"+datetime.datetime.now().strftime(\"%m%d%H%M\")+\"/\"\n",
|
||||
"LOAD_DIR = None\n",
|
||||
"\n",
|
||||
"CTN_ACTION_RANGE = 10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"√√√√√Enviroment Initialized Success√√√√√\n",
|
||||
"√√√√√Buffer Initialized Success√√√√√\n",
|
||||
"Load Model:\n",
|
||||
"PPO-Model/09051833/183540\n",
|
||||
"CONTINUOUS_SIZE 1\n",
|
||||
"DISCRETE_SIZE 5\n",
|
||||
"STATE_SIZE 29\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# initialize enviroment & buffer class\n",
|
||||
"env = aimBotEnv.makeEnv(envPath = ENV_PATH,\n",
|
||||
" workerID = WORKER_ID,\n",
|
||||
" basePort = BASE_PORT)\n",
|
||||
"epBuffer = buffer.buffer()\n",
|
||||
"\n",
|
||||
"STATE_SIZE = env.STATE_SIZE\n",
|
||||
"CONTINUOUS_SIZE = env.CONTINUOUS_SIZE\n",
|
||||
"DISCRETE_SIZE = env.DISCRETE_SIZE\n",
|
||||
"s,reward,done,loadDir = env.getSteps()\n",
|
||||
"\n",
|
||||
"# check load model or not\n",
|
||||
"if(np.any(loadDir == 0)):\n",
|
||||
" # create a new model\n",
|
||||
" print(\"No loadDir specified,Create a New Model\")\n",
|
||||
" LOAD_DIR = None\n",
|
||||
"else:\n",
|
||||
" # load model\n",
|
||||
" loadDirDateSTR = str(int(loadDir[0]))\n",
|
||||
" loadDirTimeSTR = str(int(loadDir[1]))\n",
|
||||
" if len(loadDirDateSTR)!=8:\n",
|
||||
" # fill lost 0 while converse float to string\n",
|
||||
" for _ in range(8 - len(loadDirDateSTR)):\n",
|
||||
" loadDirDateSTR = \"0\" + loadDirDateSTR\n",
|
||||
" if len(loadDirTimeSTR)!=6:\n",
|
||||
" # fill lost 0 while converse float to string\n",
|
||||
" for _ in range(6 - len(loadDirTimeSTR)):\n",
|
||||
" loadDirTimeSTR = \"0\" + loadDirTimeSTR\n",
|
||||
" LOAD_DIR = \"PPO-Model/\"+loadDirDateSTR+\"/\"+loadDirTimeSTR\n",
|
||||
" print(\"Load Model:\")\n",
|
||||
" print(LOAD_DIR)\n",
|
||||
"\n",
|
||||
"print(\"CONTINUOUS_SIZE\",CONTINUOUS_SIZE)\n",
|
||||
"print(\"DISCRETE_SIZE\",DISCRETE_SIZE)\n",
|
||||
"print(\"STATE_SIZE\",STATE_SIZE)\n",
|
||||
"\n",
|
||||
"disActShape = [3,3,2]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def actToKey(disAct1,disAct2,disAct3,conAct):\n",
|
||||
" kW = 0\n",
|
||||
" kS = 0\n",
|
||||
" kA = 0\n",
|
||||
" kD = 0\n",
|
||||
" mouseShoot = 0\n",
|
||||
" if disAct1 == 0:\n",
|
||||
" kW = 0\n",
|
||||
" kS = 1\n",
|
||||
" elif disAct1 == 1:\n",
|
||||
" kW = 0\n",
|
||||
" kS = 0\n",
|
||||
" elif disAct1 == 2:\n",
|
||||
" kW = 1\n",
|
||||
" kS = 0\n",
|
||||
" if disAct2 == 0:\n",
|
||||
" kA = 0\n",
|
||||
" kD = 1\n",
|
||||
" elif disAct2 == 1:\n",
|
||||
" kA = 0\n",
|
||||
" kD = 0\n",
|
||||
" elif disAct2 == 2:\n",
|
||||
" kA = 1\n",
|
||||
" kD = 0\n",
|
||||
" mouseShoot = disAct3\n",
|
||||
" return kW,kS,kA,kD,mouseShoot,conAct"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"EP 0 START\n",
|
||||
"√√√√√Buffer Initialized Success√√√√√\n",
|
||||
"++++++++++++++++++++++++++++++++++++\n",
|
||||
"++++++++++++Model Loaded++++++++++++\n",
|
||||
"PPO-Model/09051833/183540\n",
|
||||
"++++++++++++++++++++++++++++++++++++\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3474: RuntimeWarning: Mean of empty slice.\n",
|
||||
" return _methods._mean(a, axis=axis, dtype=dtype,\n",
|
||||
"c:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
|
||||
" ret = ret.dtype.type(ret / rcount)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"A_Loss: -43.132581075032554 C_Loss: 4025.360986328125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1184751.5479166666 C_Loss: 1939213379930.6667\n",
|
||||
"Model's Weights Saved\n",
|
||||
"New Record! Save NN -37.67000053450465\n",
|
||||
"EP 1 START\n",
|
||||
"A_Loss: 14251923.066666666 C_Loss: 40720630.7288086\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1857006.7333333334 C_Loss: 245.15654500325522\n",
|
||||
"Model's Weights Saved\n",
|
||||
"New Record! Save NN -13.100000601261854\n",
|
||||
"EP 2 START\n",
|
||||
"A_Loss: -1.0014899969100952 C_Loss: 71.29023424784343\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: -0.31318608721097313 C_Loss: 8.897234630584716\n",
|
||||
"EP 3 START\n",
|
||||
"A_Loss: 566053.9979166667 C_Loss: 1986013.3489705403\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 0.6021817366282145 C_Loss: 6.035458342234294\n",
|
||||
"EP 4 START\n",
|
||||
"A_Loss: 169656457.6 C_Loss: 4317.756831359863\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 22155934.0 C_Loss: 63.52870483398438\n",
|
||||
"EP 5 START\n",
|
||||
"A_Loss: 0.2413090835014979 C_Loss: 10.40585823059082\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 0.12157159547011058 C_Loss: 14.347647285461425\n",
|
||||
"EP 6 START\n",
|
||||
"A_Loss: 215977770.66666666 C_Loss: 190435277.5966268\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 213349568.0 C_Loss: 1620.733740234375\n",
|
||||
"EP 7 START\n",
|
||||
"A_Loss: 100149825.6 C_Loss: 1569803.3794799806\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 171062395.73333332 C_Loss: 34872400594.933334\n",
|
||||
"Model's Weights Saved\n",
|
||||
"New Record! Save NN 4.119999349117279\n",
|
||||
"EP 8 START\n",
|
||||
"A_Loss: 0.7038820425669352 C_Loss: 31.980174128214518\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 0.2687960028648376 C_Loss: 6.94128991762797\n",
|
||||
"EP 9 START\n",
|
||||
"A_Loss: 0.1451285809278488 C_Loss: 3.5754743576049806\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 0.18081151247024535 C_Loss: 3.902424764633179\n",
|
||||
"EP 10 START\n",
|
||||
"A_Loss: 29139865.6 C_Loss: 174927458.90618488\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 16942552.4 C_Loss: 134.21190592447917\n",
|
||||
"EP 11 START\n",
|
||||
"A_Loss: 53790223.46666667 C_Loss: 647.0605305989583\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 27695839.866666667 C_Loss: 343.9340047200521\n",
|
||||
"EP 12 START\n",
|
||||
"A_Loss: 65695720.8 C_Loss: 61225647615.0198\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 0.07575627962748209 C_Loss: 10.41986296971639\n",
|
||||
"EP 13 START\n",
|
||||
"A_Loss: 51573392.266666666 C_Loss: 212022811.82775268\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 22073133.866666667 C_Loss: 61.18818028767904\n",
|
||||
"EP 14 START\n",
|
||||
"A_Loss: 0.13529965033133826 C_Loss: 0.966444210211436\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 0.1176165262858073 C_Loss: 0.5141626675923665\n",
|
||||
"EP 15 START\n",
|
||||
"A_Loss: 0.5104357699553171 C_Loss: 25.79277165730794\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 15181718.0 C_Loss: 9111658759031.467\n",
|
||||
"EP 16 START\n",
|
||||
"A_Loss: 30694314.533333335 C_Loss: 62842582.26665497\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 17702280.066666666 C_Loss: 50.18375091552734\n",
|
||||
"EP 17 START\n",
|
||||
"A_Loss: 0.08570613314708074 C_Loss: 1.7101642807324728\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 14154208.066666666 C_Loss: 248.65523885091145\n",
|
||||
"EP 18 START\n",
|
||||
"A_Loss: 51043696.8 C_Loss: 39972927.18101196\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 24850118.0 C_Loss: 6779725.618615722\n",
|
||||
"EP 19 START\n",
|
||||
"A_Loss: 75697662.4 C_Loss: 505.26551717122396\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 64376180.266666666 C_Loss: 299.2391876220703\n",
|
||||
"EP 20 START\n",
|
||||
"A_Loss: 55954064.8 C_Loss: 234.9048828125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 57271699.733333334 C_Loss: 59254958.60783488\n",
|
||||
"EP 21 START\n",
|
||||
"A_Loss: 124046376.53333333 C_Loss: 2737571204155.7334\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 31774753.866666667 C_Loss: 24325611279.066666\n",
|
||||
"EP 22 START\n",
|
||||
"A_Loss: 7490019498.666667 C_Loss: 245487346653.86667\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 186999600.0 C_Loss: 36627312369.066666\n",
|
||||
"EP 23 START\n",
|
||||
"A_Loss: 4297551769.6 C_Loss: 1184376194184.5334\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 122982910044842.67 C_Loss: 7331505971746.134\n",
|
||||
"EP 24 START\n",
|
||||
"A_Loss: 2682021.3583333334 C_Loss: 2341072196027.7334\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1469103463901.8667 C_Loss: 113593101.86666666\n",
|
||||
"EP 25 START\n",
|
||||
"A_Loss: 76648460.0 C_Loss: 14498072.481510418\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 5800877.766666667 C_Loss: 22134.297265625\n",
|
||||
"EP 26 START\n",
|
||||
"A_Loss: 34429532.53333333 C_Loss: 46926.630208333336\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 212612493.86666667 C_Loss: 23176.368880208334\n",
|
||||
"EP 27 START\n",
|
||||
"A_Loss: 3207813239.4666667 C_Loss: 41575.61041666667\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 38858107.86666667 C_Loss: 15486.2171875\n",
|
||||
"EP 28 START\n",
|
||||
"A_Loss: -58850.48046875 C_Loss: 14550529378106.69\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 445133260.8 C_Loss: 1749447703.4604166\n",
|
||||
"EP 29 START\n",
|
||||
"A_Loss: 80796092.8 C_Loss: 8523564.864453126\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 34027046.266666666 C_Loss: 3319.6830078125\n",
|
||||
"Model's Weights Saved\n",
|
||||
"New Record! Save NN 4.589999713003635\n",
|
||||
"EP 30 START\n",
|
||||
"A_Loss: 118412868.26666667 C_Loss: 23715976.025211588\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 94927646.93333334 C_Loss: 1067.18818359375\n",
|
||||
"EP 31 START\n",
|
||||
"A_Loss: 338770097.06666666 C_Loss: 3585200.7686075848\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1479514146.1333334 C_Loss: 10348.122916666667\n",
|
||||
"Model's Weights Saved\n",
|
||||
"New Record! Save NN 38.5099995136261\n",
|
||||
"EP 32 START\n",
|
||||
"A_Loss: 119593341.86666666 C_Loss: 784.9417297363282\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: -3.2093143463134766 C_Loss: 180.5671844482422\n",
|
||||
"EP 33 START\n",
|
||||
"A_Loss: 149599549.86666667 C_Loss: 206.16288146972656\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 5682631.7 C_Loss: 17.62857920328776\n",
|
||||
"EP 34 START\n",
|
||||
"A_Loss: 231213670.4 C_Loss: 272.8009948730469\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 167307104.0 C_Loss: 188.52051798502603\n",
|
||||
"EP 35 START\n",
|
||||
"A_Loss: 204825578.66666666 C_Loss: 335.75029296875\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 83991501.33333333 C_Loss: 451.3644246419271\n",
|
||||
"EP 36 START\n",
|
||||
"A_Loss: 0.6501724402109782 C_Loss: 38.280514017740884\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 175042737.06666666 C_Loss: 256.6915344238281\n",
|
||||
"EP 37 START\n",
|
||||
"A_Loss: 461190880.0 C_Loss: 717.4720499674479\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 173152186.66666666 C_Loss: 701.6123962402344\n",
|
||||
"EP 38 START\n",
|
||||
"A_Loss: 188070140.8 C_Loss: 1611377609664.9072\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 512005579391093.7 C_Loss: 86553492717568.0\n",
|
||||
"EP 39 START\n",
|
||||
"A_Loss: 121553.57962239583 C_Loss: 53618558110.88125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 10892.0080078125 C_Loss: 132429.940625\n",
|
||||
"EP 40 START\n",
|
||||
"A_Loss: 15404.601106770833 C_Loss: 107106.1421875\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1822.5163004557292 C_Loss: 182031.73854166668\n",
|
||||
"EP 41 START\n",
|
||||
"A_Loss: 599.1890706380208 C_Loss: 76118.35729166666\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1033.1336588541667 C_Loss: 54971.41354166667\n",
|
||||
"EP 42 START\n",
|
||||
"A_Loss: 2602.054020182292 C_Loss: 227568.73541666666\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: -3.2474422613779703 C_Loss: 155.22516682942708\n",
|
||||
"EP 43 START\n",
|
||||
"A_Loss: 1994.366455078125 C_Loss: 149748.48125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 7441.857747395833 C_Loss: 196344.43958333333\n",
|
||||
"EP 44 START\n",
|
||||
"A_Loss: 19441.808723958333 C_Loss: 197749.58020833333\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1789.1592529296875 C_Loss: 25795.180598958334\n",
|
||||
"EP 45 START\n",
|
||||
"A_Loss: 6888.085221354167 C_Loss: 149084.575\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1102350.757421875 C_Loss: 180759.19166666668\n",
|
||||
"EP 46 START\n",
|
||||
"A_Loss: 536.078184000651 C_Loss: 89909.24375\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 794341992174.4 C_Loss: 59005630871824.9\n",
|
||||
"EP 47 START\n",
|
||||
"A_Loss: 114184075127.46666 C_Loss: 74263479521553.06\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1293861572.3333333 C_Loss: 1474176015837.8667\n",
|
||||
"EP 48 START\n",
|
||||
"A_Loss: 1143.1249674479166 C_Loss: 19221068817.411232\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 855.1484252929688 C_Loss: 28676.431770833333\n",
|
||||
"EP 49 START\n",
|
||||
"A_Loss: 1198.7060546875 C_Loss: 41875.48359375\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1215.0390218098958 C_Loss: 7167.594791666666\n",
|
||||
"EP 50 START\n",
|
||||
"A_Loss: 1840.496435546875 C_Loss: 33346.76875\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 97.68925425211589 C_Loss: 2453.3792643229167\n",
|
||||
"EP 51 START\n",
|
||||
"A_Loss: 2179.5446614583334 C_Loss: 25573.869270833333\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 3547.454052734375 C_Loss: 15107.698307291666\n",
|
||||
"EP 52 START\n",
|
||||
"A_Loss: 513.1327128092448 C_Loss: 23233.316276041667\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 2024.9339111328125 C_Loss: 10457.81875\n",
|
||||
"EP 53 START\n",
|
||||
"A_Loss: 321.8419494628906 C_Loss: 10830.605924479167\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: -8.38951981862386 C_Loss: 2241.023612467448\n",
|
||||
"EP 54 START\n",
|
||||
"A_Loss: 1454.901765950521 C_Loss: 16337.481966145833\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 3069.5313313802085 C_Loss: 4450.756380208333\n",
|
||||
"EP 55 START\n",
|
||||
"A_Loss: 444.8317138671875 C_Loss: 15201.106184895832\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1159.2985107421875 C_Loss: 35525.82317708333\n",
|
||||
"EP 56 START\n",
|
||||
"A_Loss: 1397.7694742838542 C_Loss: 8600.898404947917\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 160.49285074869792 C_Loss: 3749.308203125\n",
|
||||
"EP 57 START\n",
|
||||
"A_Loss: -23.67707913716634 C_Loss: 8787.21162109375\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 446.2368998209635 C_Loss: 8949.849674479166\n",
|
||||
"EP 58 START\n",
|
||||
"A_Loss: 1443.8223063151042 C_Loss: 4806.931754557291\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1028.3776000976563 C_Loss: 10080.125455729167\n",
|
||||
"EP 59 START\n",
|
||||
"A_Loss: 979.55380859375 C_Loss: 18168.035872395834\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 2799.885953776042 C_Loss: 14584.458203125\n",
|
||||
"EP 60 START\n",
|
||||
"A_Loss: 83247.30833333333 C_Loss: 61687910677571.76\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 81.43709259033203 C_Loss: 47630220.59791667\n",
|
||||
"EP 61 START\n",
|
||||
"A_Loss: 5032.870377604167 C_Loss: 43944.79453125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 9.13815720876058 C_Loss: 2573.8714192708335\n",
|
||||
"EP 62 START\n",
|
||||
"A_Loss: 523.7818908691406 C_Loss: 8441.000618489583\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1593.0845377604167 C_Loss: 17598.201953125\n",
|
||||
"EP 63 START\n",
|
||||
"A_Loss: 27.476329803466797 C_Loss: 5594.57646484375\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 25.40858777364095 C_Loss: 11878.195572916668\n",
|
||||
"EP 64 START\n",
|
||||
"A_Loss: 3347.620393880208 C_Loss: 15555.9111328125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 25.35328992207845 C_Loss: 7923.337727864583\n",
|
||||
"EP 65 START\n",
|
||||
"A_Loss: 1800.478165690104 C_Loss: 36845.799479166664\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 544.9222574869792 C_Loss: 2949.7829752604166\n",
|
||||
"EP 66 START\n",
|
||||
"A_Loss: 238.4148183186849 C_Loss: 2108.7259195963543\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 10.808652464548747 C_Loss: 1069.8261800130208\n",
|
||||
"EP 67 START\n",
|
||||
"A_Loss: 7.2742944399515785 C_Loss: 1112.0956298828125\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 686.372725423177 C_Loss: 1421.539111328125\n",
|
||||
"EP 68 START\n",
|
||||
"A_Loss: 38.04718589782715 C_Loss: 2771.959724934896\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 1133.577986653646 C_Loss: 5758.061686197917\n",
|
||||
"EP 69 START\n",
|
||||
"A_Loss: 402.1093465169271 C_Loss: 3987.9388671875\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: -1.7969589630762737 C_Loss: 2884.10732421875\n",
|
||||
"EP 70 START\n",
|
||||
"A_Loss: 1016.8031656901042 C_Loss: 6636.073665364584\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 939.9079630533854 C_Loss: 3692.443391927083\n",
|
||||
"EP 71 START\n",
|
||||
"A_Loss: -107.65687510172526 C_Loss: 21740540498583.367\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 5876.485221354166 C_Loss: 50534.828385416666\n",
|
||||
"EP 72 START\n",
|
||||
"A_Loss: 219.98338216145834 C_Loss: 2713.9967041015625\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 433.9121032714844 C_Loss: 950.6825520833333\n",
|
||||
"EP 73 START\n",
|
||||
"A_Loss: 933.4014811197917 C_Loss: 1297.3184733072917\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 2035.8281412760416 C_Loss: 3957.7666178385416\n",
|
||||
"EP 74 START\n",
|
||||
"A_Loss: 479.3147298177083 C_Loss: 1727.3513753255208\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 579.0808024088542 C_Loss: 635.3917744954427\n",
|
||||
"EP 75 START\n",
|
||||
"A_Loss: 1450.3585856119792 C_Loss: 2148.2808756510417\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 917.6628540039062 C_Loss: 1192.1792277018228\n",
|
||||
"EP 76 START\n",
|
||||
"A_Loss: 888.0876180013021 C_Loss: 1050.4912679036458\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 249.14984232584635 C_Loss: 9789.355598958333\n",
|
||||
"EP 77 START\n",
|
||||
"A_Loss: 346.7358866373698 C_Loss: 1828.4739990234375\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 15.01647071838379 C_Loss: 1649.2421305338542\n",
|
||||
"EP 78 START\n",
|
||||
"A_Loss: 2058.658439127604 C_Loss: 1996.608447265625\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 155.17740580240886 C_Loss: 575.6137990315755\n",
|
||||
"EP 79 START\n",
|
||||
"A_Loss: 1246.8288167317708 C_Loss: 1408.468212890625\n",
|
||||
"EP OVER!\n",
|
||||
"A_Loss: 175.76889241536458 C_Loss: 825.604911295573\n",
|
||||
"EP 80 START\n",
|
||||
"A_Loss: 10138501.666666666 C_Loss: 33048526138440.633\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"bestScore = 200.\n",
|
||||
"stopTrainCounter = 0\n",
|
||||
"\n",
|
||||
"totalRewardHis = []\n",
|
||||
"totalActorLossHis = []\n",
|
||||
"totalCriticLossHis = []\n",
|
||||
"epHis = []\n",
|
||||
"maxTotalReward = -99999999999\n",
|
||||
"\n",
|
||||
"for ep in range(MAX_EP):\n",
|
||||
" print(\"EP \",ep,\" START\")\n",
|
||||
" # first time run game\n",
|
||||
" s,_,_,_ = env.reset()\n",
|
||||
" if (ep == 0):\n",
|
||||
" epBuffer = buffer.buffer()\n",
|
||||
" s = s.reshape([STATE_SIZE])\n",
|
||||
" agent = PPO.PPO(stateSize=STATE_SIZE,\n",
|
||||
" disActShape=disActShape,\n",
|
||||
" conActSize=1,\n",
|
||||
" conActRange=CTN_ACTION_RANGE,\n",
|
||||
" criticLR=CRITIC_LR,\n",
|
||||
" actorLR=ACTOR_LR,\n",
|
||||
" gamma=GAMMA,\n",
|
||||
" epsilon=EPSILON,\n",
|
||||
" entropyWeight=ENTROPY_WHEIGHT,\n",
|
||||
" saveDir=SAVE_DIR,\n",
|
||||
" loadModelDir=LOAD_DIR)\n",
|
||||
" step = 0\n",
|
||||
" done = False\n",
|
||||
" stopTrainCounter -= 1\n",
|
||||
" epHis.append(ep)\n",
|
||||
" \n",
|
||||
" # reset total reward\n",
|
||||
" epTotalReward = 0\n",
|
||||
" \n",
|
||||
" # Recorder list\n",
|
||||
" epStepHis = []\n",
|
||||
" epRewardHis = []\n",
|
||||
" epActorLossHis = []\n",
|
||||
" epCriticLossHis = []\n",
|
||||
"\n",
|
||||
" while not done:\n",
|
||||
" step += 1\n",
|
||||
" if step % ACTION_INTERVAL == 0: # take action every ACTION_INTERVAL steps\n",
|
||||
" epStepHis.append(step)\n",
|
||||
" disAct1,disAct2,disAct3,conAct,predictResult = agent.chooseAction(s)\n",
|
||||
" kW, kS, kA, kD, mouseShoot, mouseMove = actToKey(disAct1,disAct2,disAct3,conAct)\n",
|
||||
" \n",
|
||||
" nextState,thisReward,done,_ = env.step(discreteActions=np.array([[kW, kS, kA, kD, mouseShoot]]),continuousActions=np.array([[mouseMove]]))\n",
|
||||
"\n",
|
||||
" epTotalReward += thisReward\n",
|
||||
" epBuffer.saveBuffers(s,[disAct1,disAct2,disAct3,conAct],thisReward)\n",
|
||||
" else:\n",
|
||||
" disActs = np.array([[0,0,0,0,0]])\n",
|
||||
" conActs = np.array([[0]])\n",
|
||||
"\n",
|
||||
" nextState,thisReward,done,_ = env.step(discreteActions=disActs,continuousActions=conActs)\n",
|
||||
" epTotalReward += thisReward\n",
|
||||
" nextState = nextState.reshape([STATE_SIZE])\n",
|
||||
" s = nextState\n",
|
||||
" \n",
|
||||
" if done:\n",
|
||||
" print(\"EP OVER!\")\n",
|
||||
" # update PPO after Batch step or GameOver\n",
|
||||
" if (step+1)%BATCH == 0 or done:\n",
|
||||
" bs = epBuffer.getStates()\n",
|
||||
" ba = epBuffer.getActions()\n",
|
||||
" br = epBuffer.getRewards()\n",
|
||||
" epBuffer.clearBuffer()\n",
|
||||
" if TRAIN:\n",
|
||||
" epActorLoss,epCriticLoss = agent.trainCritcActor(bs,ba,br,s,CRITIC_EPOCH,ACTOR_EPOCH)\n",
|
||||
" epActorLossHis.append(epActorLoss)\n",
|
||||
" epCriticLossHis.append(epCriticLoss)\n",
|
||||
" # update History Recorder\n",
|
||||
" totalActorLossHis.append(np.mean(epActorLossHis))\n",
|
||||
" totalCriticLossHis.append(np.mean(epCriticLossHis))\n",
|
||||
" totalRewardHis.append(epTotalReward)\n",
|
||||
" \n",
|
||||
" if (epTotalReward > maxTotalReward and epTotalReward != 0):\n",
|
||||
" maxTotalReward = epTotalReward\n",
|
||||
" agent.saveWeights(epTotalReward)\n",
|
||||
" print(\"New Record! Save NN\",epTotalReward)\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"aaa = 0\n",
|
||||
"aaa = 1\n",
|
||||
"aaa = 2\n",
|
||||
"aaa = 3\n",
|
||||
"aaa = 4\n",
|
||||
"aaa = 5\n",
|
||||
"aaa = 6\n",
|
||||
"aaa = 7\n",
|
||||
"aaa = 8\n",
|
||||
"aaa = 9\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"aaa = 0\n",
|
||||
"while aaa<10:\n",
|
||||
" print(\"aaa = \",aaa)\n",
|
||||
" aaa+=1"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.7 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.7"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import aimBotEnv
|
||||
import PPO
|
||||
|
||||
ENV_PATH = './Build/Aimbot-PPO'
|
||||
WORKER_ID = 100
|
||||
|
||||
MAX_EP = 1000
|
||||
EP_LENGTH = 400
|
||||
GAMMA = 0.99 # discount future reward (UP?)
|
||||
EPSILON = 0.2 # clip Ratio range[1-EPSILON,1+EPSILON]
|
||||
ACTOR_LR = 1e-5 # LR
|
||||
CRITIC_LR = 2e-5 # LR
|
||||
BATCH = 32 # learning step
|
||||
ACTOR_EPOCH = 10 # epoch
|
||||
CRITIC_EPOCH = 10 # epoch
|
||||
ENTROPY_WHEIGHT = 0.01 # sigma's entropy in Actor loss
|
||||
ACTION_INTERVAL = 1 # take action every ACTION_INTERVAL steps
|
||||
TRAIN = True
|
||||
|
||||
env = aimBotEnv.makeEnv(envPath = ENV_PATH,workerID = WORKER_ID)
|
||||
STATE_SIZE = env.STATE_SIZE
|
||||
CONTINUOUS_SIZE = env.CONTINUOUS_SIZE
|
||||
DISCRETE_SIZE = env.DISCRETE_SIZE
|
||||
|
||||
CTN_ACTION_RANGE = 2
|
||||
@@ -0,0 +1,44 @@
|
||||
import numpy as np
|
||||
|
||||
class rewardAI(object):
|
||||
def __init__(self,nonReward, shootReward, shootWithoutReadyReward, hitReward, winReward, loseReward, killReward):
|
||||
self.nonReward = nonReward
|
||||
self.shootReward = shootReward
|
||||
self.shootWithoutReadyReward = shootWithoutReadyReward
|
||||
self.hitReward = hitReward
|
||||
self.winReward = winReward
|
||||
self.loseReward = loseReward
|
||||
self.killReward = killReward
|
||||
|
||||
def getRewards(self,remainTime):
|
||||
nonR = self.getnonReward()
|
||||
shootR = self.getshootReward()
|
||||
shootWithoutReadyR = self.getshootWithoutReadyReward()
|
||||
hitR = self.gethitReward()
|
||||
winR = self.getwinReward(remainTime)
|
||||
loseR = self.getloseReward()
|
||||
killR = self.getkillReward(remainTime)
|
||||
|
||||
rewards = np.array([[nonR,
|
||||
shootR,
|
||||
shootWithoutReadyR,
|
||||
hitR,
|
||||
winR,
|
||||
loseR,
|
||||
killR]], dtype=np.float)
|
||||
return rewards
|
||||
|
||||
def getnonReward (self):
|
||||
return self.nonReward
|
||||
def getshootReward(self):
|
||||
return self.shootReward
|
||||
def getshootWithoutReadyReward(self):
|
||||
return self.shootWithoutReadyReward
|
||||
def gethitReward(self):
|
||||
return self.hitReward
|
||||
def getwinReward(self,time):
|
||||
return (self.winReward + time)
|
||||
def getloseReward(self):
|
||||
return self.loseReward
|
||||
def getkillReward(self,time):
|
||||
return (self.killReward + time)
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[INFO] Connected to Unity environment with package version 2.0.0 and communication version 1.5.0\n",
|
||||
"[INFO] Connected new brain: AKMAgent?team=0\n",
|
||||
"[WARNING] uint8_visual was set to true, but visual observations are not in use. This setting will not have any effect.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "UnityGymException",
|
||||
"evalue": "The gym wrapper does not provide explicit support for both discrete and continuous actions.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mUnityGymException\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10504/1419431386.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'__main__'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m \u001b[0mmain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
|
||||
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10504/1419431386.py\u001b[0m in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mmain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 9\u001b[0m \u001b[0munity_env\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mUnityEnvironment\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mENV_PATH\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mseed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mside_channels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mworker_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mWORKER_ID\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbase_port\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBASE_PORT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m \u001b[0menv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mUnityToGymWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munity_env\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muint8_visual\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mallow_multiple_obs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 11\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[1;32mc:\\Users\\UCUNI\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\gym_unity\\envs\\__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, unity_env, uint8_visual, flatten_branched, allow_multiple_obs, action_space_seed)\u001b[0m\n\u001b[0;32m 128\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_action_space\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mspaces\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBox\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mhigh\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhigh\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 129\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 130\u001b[1;33m raise UnityGymException(\n\u001b[0m\u001b[0;32m 131\u001b[0m \u001b[1;34m\"The gym wrapper does not provide explicit support for both discrete \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 132\u001b[0m \u001b[1;34m\"and continuous actions.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[1;31mUnityGymException\u001b[0m: The gym wrapper does not provide explicit support for both discrete and continuous actions."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from mlagents_envs.environment import UnityEnvironment\n",
|
||||
"from gym_unity.envs import UnityToGymWrapper\n",
|
||||
"\n",
|
||||
"ENV_PATH = './Build/Aimbot-PPO'\n",
|
||||
"WORKER_ID = 2\n",
|
||||
"BASE_PORT = 2002\n",
|
||||
"\n",
|
||||
"def main():\n",
|
||||
" unity_env = UnityEnvironment(file_name=ENV_PATH,seed = 1,side_channels=[],worker_id = WORKER_ID,base_port=BASE_PORT)\n",
|
||||
" env = UnityToGymWrapper(unity_env, uint8_visual=True,allow_multiple_obs=True)\n",
|
||||
" env.reset()\n",
|
||||
" for _ in range(1000):\n",
|
||||
" env.render()\n",
|
||||
" o,r,d,_ = env.step(env.action_space.sample()) #random action\n",
|
||||
" if d:\n",
|
||||
" env.reset()\n",
|
||||
" env.close() \n",
|
||||
" \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if __name__ == '__main__':\n",
|
||||
" main()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.7 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.7"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array([[ 1, 2, 3, 7, 8, 9, 10]])"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"state = np.array([[1,2,3],[1,2,3]])\n",
|
||||
"aaa = np.array([[123]])\n",
|
||||
"\n",
|
||||
"state[:,-1]\n",
|
||||
"\n",
|
||||
"np.append([[1, 2, 3]], [[7, 8, 9, 10]], axis=1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"aa = tf.constant([[1,2,3],[1,2,3],[1,2,3],[1,2,3]])\n",
|
||||
"bb = tf.constant([1,2,3,4])\n",
|
||||
"\n",
|
||||
"print(tf.expand_dims(bb,axis=1))\n",
|
||||
"\n",
|
||||
"cc = tf.math.multiply(aa,tf.expand_dims(bb,axis = 1))\n",
|
||||
"\n",
|
||||
"print(cc)\n",
|
||||
"print(tf.shape(aa))\n",
|
||||
"\n",
|
||||
"print(aa[:,2:3])\n",
|
||||
"\n",
|
||||
"aa = tf.constant([1.0,2.0,3.0,np.nan])\n",
|
||||
"print(np.any(tf.math.is_nan(aa)))\n",
|
||||
"if np.any(tf.math.is_nan(aa)):\n",
|
||||
" print('true')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"prob = tf.constant([0.3,0.3,0.])\n",
|
||||
"\n",
|
||||
"entropy = tf.reduce_mean(tf.math.negative(tf.math.multiply(prob,tf.math.log(prob))))\n",
|
||||
"\n",
|
||||
"print(entropy)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"int 23\n",
|
||||
"str twenty three\n",
|
||||
"bool True\n",
|
||||
"error\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from functools import singledispatchmethod\n",
|
||||
"class person:\n",
|
||||
" @singledispatchmethod\n",
|
||||
" def age(self,arg):\n",
|
||||
" print(\"error\")\n",
|
||||
" @age.register(int)\n",
|
||||
" def _(self,arg:int):\n",
|
||||
" print(\"int\",arg)\n",
|
||||
" @age.register(str)\n",
|
||||
" def _(self,arg:str):\n",
|
||||
" print(\"str\",arg)\n",
|
||||
" @age.register(bool)\n",
|
||||
" def _(self,arg:bool):\n",
|
||||
" print(\"bool\",arg)\n",
|
||||
"\n",
|
||||
"p = person()\n",
|
||||
"p.age(23) # int\n",
|
||||
"p.age('twenty three') # str\n",
|
||||
"p.age(True) # bool\n",
|
||||
"p.age(['23']) # list\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"import tensorflow.keras.layers as layers\n",
|
||||
"\n",
|
||||
"def conv_bn_relu(inputs, chs, reps):\n",
|
||||
" x = inputs\n",
|
||||
" for i in range(reps):\n",
|
||||
" x = layers.Conv2D(chs, 3, padding=\"same\")(x)\n",
|
||||
" x = layers.BatchNormalization()(x)\n",
|
||||
" x = layers.ReLU()(x)\n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
"def create_model():\n",
|
||||
" inputs = layers.Input((32, 32, 3))\n",
|
||||
" x = conv_bn_relu(inputs, 64, 3)\n",
|
||||
" x = layers.AveragePooling2D(2)(x)\n",
|
||||
" x = conv_bn_relu(x, 128, 3)\n",
|
||||
" x = layers.AveragePooling2D(2)(x)\n",
|
||||
" x = conv_bn_relu(x, 256, 3)\n",
|
||||
" x = layers.GlobalAveragePooling2D()(x)\n",
|
||||
" x = layers.Dense(10, activation=\"softmax\")(x)\n",
|
||||
" return tf.keras.models.Model(inputs, x)\n",
|
||||
"\n",
|
||||
"def perprocess(img, label):\n",
|
||||
" img = tf.cast(img, tf.float32) / 255.0\n",
|
||||
" label = tf.cast(label, tf.float32)\n",
|
||||
" return img, label\n",
|
||||
"\n",
|
||||
"def train():\n",
|
||||
" (X_train, y_train), (X_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
|
||||
" trainset = tf.data.Dataset.from_tensor_slices((X_train, y_train)\n",
|
||||
" ).map(perprocess).shuffle(4096).batch(128).repeat().prefetch(50)\n",
|
||||
"\n",
|
||||
" model = create_model()\n",
|
||||
" model.compile(\"adam\", \"sparse_categorical_crossentropy\", [\"acc\"])\n",
|
||||
"\n",
|
||||
" model.fit(trainset, steps_per_epoch=50000//128, epochs=1) \n",
|
||||
" # 'Adam/conv2d/kernel/m:0' shape=(3, 3, 3, 64) \n",
|
||||
" print(model.optimizer.weights[1][0, 0, 0,:10])\n",
|
||||
" # <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 3, 64)\n",
|
||||
" print(model.weights[0][0, 0, 0,:10])\n",
|
||||
"\n",
|
||||
" model.save_weights(\"model_tf.ckpt\", save_format=\"tf\") # デフォルト\n",
|
||||
" model.save_weights(\"model_h5.h5\", save_format=\"h5\") "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"390/390 [==============================] - 28s 59ms/step - loss: 1.2548 - acc: 0.5441\n",
|
||||
"tf.Tensor(\n",
|
||||
"[ 0.00190057 -0.00765918 0.00163367 0.00782851 0.02600338 0.00516749\n",
|
||||
" -0.00424899 0.01562062 -0.0022073 -0.00355565], shape=(10,), dtype=float32)\n",
|
||||
"tf.Tensor(\n",
|
||||
"[ 0.07978954 -0.04595745 -0.03745254 -0.03701654 0.03296526 -0.11328737\n",
|
||||
" -0.10719797 0.00874998 0.0226855 0.02288487], shape=(10,), dtype=float32)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"train()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tf.Tensor(\n",
|
||||
"[ 0.07978954 -0.04595745 -0.03745254 -0.03701654 0.03296526 -0.11328737\n",
|
||||
" -0.10719797 0.00874998 0.0226855 0.02288487], shape=(10,), dtype=float32)\n",
|
||||
"tf.Tensor(\n",
|
||||
"[ 0.00190057 -0.00765918 0.00163367 0.00782851 0.02600338 0.00516749\n",
|
||||
" -0.00424899 0.01562062 -0.0022073 -0.00355565], shape=(10,), dtype=float32)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def load_tf_w_zero_grad():\n",
|
||||
" model = create_model()\n",
|
||||
" model.compile(\"adam\", \"sparse_categorical_crossentropy\", [\"acc\"])\n",
|
||||
"\n",
|
||||
" zero_grad = [tf.zeros_like(x) for x in model.weights]\n",
|
||||
" model.optimizer.apply_gradients(zip(zero_grad, model.weights))\n",
|
||||
"\n",
|
||||
" model.load_weights(\"model_tf.ckpt\")\n",
|
||||
" # これでようやくオプティマイザーの値も同一になる\n",
|
||||
" print(model.weights[0][0, 0, 0,:10])\n",
|
||||
" print(model.optimizer.weights[1][0, 0, 0,:10])\n",
|
||||
"load_tf_w_zero_grad()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"True"
|
||||
]
|
||||
},
|
||||
"execution_count": 20,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"a = np.array([10,20,30,0])\n",
|
||||
"\n",
|
||||
"np.any(a == 0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"4"
|
||||
]
|
||||
},
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"asd = \"adsf\"\n",
|
||||
"len(asd)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.7 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.7"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user