Archive all tensorflow agents and env

archive all TF py&ipynb turn face to pytorch.
2022-10-26 03:15:37 +09:00
parent cf2e4cee2c
commit 742529ccd7
18 changed files with 62 additions and 94 deletions
@@ -0,0 +1,203 @@
+import tensorflow as tf
+import numpy as np
+from numpy import ndarray
+
+from PPO import PPO
+from tensorflow import keras
+from tensorflow.keras import layers
+from tensorflow.keras import optimizers
+
+from GAILConfig import GAILConfig
+
+EPS = 1e-6
+
+
+class GAIL(object):
+    def __init__(
+        self,
+        stateSize: int,
+        disActShape: list,
+        conActSize: int,
+        conActRange: float,
+        gailConfig: GAILConfig,
+    ):
+        if disActShape == [0]:
+            # non dis action output
+            self.disActSize = 0
+            self.disOutputSize = 0
+        else:
+            try:
+                if np.any(np.array(disActShape) <= 1):
+                    raise ValueError(
+                        "disActShape error,disActShape should greater than 1 but get", disActShape
+                    )
+            except ValueError:
+                raise
+            self.disActSize = len(disActShape)
+            self.disOutputSize = sum(disActShape)
+
+        self.stateSize = stateSize
+        self.disActShape = disActShape
+        self.conActSize = conActSize
+        self.conActRange = conActRange
+
+        self.totalActSize = self.disActSize + conActSize
+        self.discrimInputSize = stateSize + self.totalActSize
+        self.discriminatorNNShape = gailConfig.discrimNNShape
+        self.discrimLR = gailConfig.discrimLR
+        self.discrimTrainEpochs = gailConfig.discrimTrainEpochs
+        self.discrimSaveDir = gailConfig.discrimSaveDir
+        self.ppoConfig = gailConfig.ppoConfig
+
+        self.ppo = PPO(stateSize, disActShape, conActSize, conActRange, self.ppoConfig)
+        self.discriminator = self.buildDiscriminatorNet(True)
+
+    def buildDiscriminatorNet(self, compileModel: bool):
+        # -----------Input Layers-----------
+        stateInput = layers.Input(shape=(self.discrimInputSize,), name="stateInput")
+
+        # -------Intermediate layers--------
+        interLayers = []
+        interLayersIndex = 0
+        for neuralUnit in self.discriminatorNNShape:
+            thisLayerName = "dense" + str(interLayersIndex)
+            if interLayersIndex == 0:
+                interLayers.append(
+                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
+                )
+            else:
+                interLayers.append(
+                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
+                )
+            interLayersIndex += 1
+
+        # ----------Output Layers-----------
+        output = layers.Dense(1, activation="sigmoid")(interLayers[-1])
+
+        # ----------Model Compile-----------
+        model = keras.Model(inputs=stateInput, outputs=output)
+        if compileModel:
+            criticOPT = optimizers.Adam(learning_rate=self.discrimLR)
+            model.compile(optimizer=criticOPT, loss=self.discrimLoss())
+        return model
+
+    def discrimLoss(self):
+        def loss(y_true, y_pred):
+            """discriminator loss function
+
+            Args:
+                y_true (tf.constant): demo trajectory
+                y_pred (tf.constant): agent trajectory predict value
+
+            Returns:
+                _type_: _description_
+            """
+            demoP = self.discriminator(y_true)
+            agentLoss = tf.negative(tf.reduce_mean(tf.math.log(1.0 - y_pred + EPS)))
+            demoLoss = tf.negative(tf.reduce_mean(tf.math.log(demoP + EPS)))
+            loss = agentLoss + demoLoss
+            return loss
+
+        return loss
+
+    def inference(self, states: ndarray, actions: ndarray):
+        """discriminator predict result
+
+        Args:
+            states (ndarray): states
+            actions (ndarray): actions
+
+        Returns:
+            tf.constant: discrim predict result
+        """
+        # check dimention
+        if states.ndim != 2:
+            stateNum = int(len(states) / self.stateSize)
+            states = states.reshape([stateNum, self.stateSize])
+        if actions.ndim != 2:
+            actionsNum = int(len(actions) / self.totalActSize)
+            actions = actions.reshape([actionsNum, self.totalActSize])
+
+        thisTrajectory = np.append(states, actions, axis=1)
+        discrimPredict = self.discriminator(thisTrajectory)
+        return discrimPredict
+
+    def discriminatorACC(
+        self, demoStates: ndarray, demoActions: ndarray, agentStates: ndarray, agentActions: ndarray
+    ):
+        demoAcc = np.mean(self.inference(demoStates, demoActions))
+        agentAcc = np.mean(self.inference(agentStates, agentActions))
+        return demoAcc, agentAcc
+
+    def trainDiscriminator(
+        self,
+        demoStates: ndarray,
+        demoActions: ndarray,
+        agentStates: ndarray,
+        agentActions: ndarray,
+        epochs: int = None,
+    ):
+        """train Discriminator
+
+        Args:
+            demoStates (ndarray): expert states
+            demoActions (ndarray): expert actions
+            agentStates (ndarray): agentPPO generated states
+            agentActions (ndarray): agentPPO generated actions
+            epoch (int): epoch times
+
+        Returns:
+            tf.constant: all losses array
+        """
+        if epochs == None:
+            epochs = self.discrimTrainEpochs
+        demoTrajectory = np.append(demoStates, demoActions, axis=1)
+        agentTrajectory = np.append(agentStates, agentActions, axis=1)
+        his = self.discriminator.fit(x=agentTrajectory, y=demoTrajectory, epochs=epochs, verbose=0)
+
+        demoAcc = np.mean(self.inference(demoStates, demoActions))
+        agentAcc = np.mean(self.inference(agentStates, agentActions))
+        return his.history["loss"], demoAcc, 1 - agentAcc
+
+    def getActions(self, state: ndarray):
+        """Agent choose action to take
+
+        Args:
+            state (ndarray): enviroment state
+
+        Returns:
+            np.array:
+                actions,
+                    actions list,2dims like [[0],[1],[1.5]]
+                predictResult,
+                    actor NN predict Result output
+        """
+        actions, predictResult = self.ppo.chooseAction(state)
+        return actions, predictResult
+
+    def trainPPO(
+        self,
+        states: ndarray,
+        oldActorResult: ndarray,
+        actions: ndarray,
+        newRewards: ndarray,
+        dones: ndarray,
+        nextState: ndarray,
+        epochs: int = None,
+    ):
+        criticV = self.ppo.getCriticV(states)
+        discountedR = self.ppo.discountReward(nextState, criticV, dones, newRewards)
+        advantage = self.ppo.getGAE(discountedR, criticV)
+        criticLosses = self.ppo.trainCritic(states, discountedR, epochs)
+        actorLosses = self.ppo.trainActor(states, oldActorResult, actions, advantage, epochs)
+        return actorLosses, criticLosses
+
+    def saveWeights(self, score: float):
+        saveDir = self.discrimSaveDir + "discriminator/discriminator.ckpt"
+        self.discriminator.save_weights(saveDir, save_format="tf")
+        print("GAIL Model's Weights Saved")
+        self.ppo.saveWeights(score=score)
+
+    def generateAction(self, states: ndarray):
+        act, actorP = self.ppo.chooseAction(states)
+        return act, actorP
@@ -0,0 +1,24 @@
+import datetime
+from typing import NamedTuple
+
+from PPOConfig import PPOConfig
+
+
+class GAILConfig(NamedTuple):
+    discrimNNShape: list = [128, 64]
+    discrimLR: float = 1e-3
+    discrimTrainEpochs: int = 8
+    discrimSaveDir: str = "GAIL-Model/" + datetime.datetime.now().strftime("%m%d-%H%M") + "/"
+
+    ppoConfig: PPOConfig = PPOConfig(
+        NNShape=[128, 64],
+        actorLR=2e-3,
+        criticLR=2e-3,
+        gamma=0.99,
+        lmbda=0.95,
+        clipRange=0.20,
+        entropyWeight=1e-2,
+        trainEpochs=8,
+        saveDir="GAIL-Model/" + datetime.datetime.now().strftime("%m%d-%H%M") + "/",
+        loadModelDir=None,
+    )
@@ -0,0 +1,53 @@
+import matplotlib.pyplot as plt
+
+DarkBlue = "#011627"
+DarkWhite = "#c9d2df"
+
+
+class GAILHistory(object):
+    def __init__(self):
+        self.meanRewards = []
+        self.discrimLosses = []
+        self.actorLosses = []
+        self.criticLosses = []
+        self.demoAccs = []
+        self.agentAccs = []
+        self.averageEntropys = []
+        self.discrimRewards = []
+
+    def saveHis(
+        self, rewards, dLosses, aLosses, cLosses, demoAcc, agentAcc, averageEntropy, discrimReward
+    ):
+        self.meanRewards.extend([rewards])
+        self.discrimLosses.extend(dLosses)
+        self.actorLosses.extend(aLosses)
+        self.criticLosses.extend(cLosses)
+        self.demoAccs.extend([demoAcc])
+        self.agentAccs.extend([agentAcc])
+        self.averageEntropys.extend([averageEntropy])
+        self.discrimRewards.extend(discrimReward)
+
+    def drawHis(self):
+        def setSubFig(subFig, data, title):
+            subFig.set_facecolor(DarkBlue)
+            subFig.tick_params(colors=DarkWhite)
+            subFig.spines["top"].set_color(DarkWhite)
+            subFig.spines["bottom"].set_color(DarkWhite)
+            subFig.spines["left"].set_color(DarkWhite)
+            subFig.spines["right"].set_color(DarkWhite)
+            subFig.plot(range(len(data)), data, color=DarkWhite, label=title)
+            subFig.set_title(title, color=DarkWhite)
+
+        fig, ((ax1, ax2), (ax3, ax4), (ax5, ax6), (ax7, ax8)) = plt.subplots(
+            4, 2, figsize=(21, 13), facecolor=DarkBlue
+        )
+        plt.tick_params()
+        setSubFig(ax1, self.meanRewards, "meanRewards")
+        setSubFig(ax2, self.discrimLosses, "discrimLosses")
+        setSubFig(ax3, self.demoAccs, "demoAccs")
+        setSubFig(ax4, self.actorLosses, "actorLosses")
+        setSubFig(ax5, self.agentAccs, "agentAccs")
+        setSubFig(ax6, self.criticLosses, "criticLosses")
+        setSubFig(ax7, self.averageEntropys, "averageEntropys")
+        setSubFig(ax8, self.discrimRewards, "discrimRewards")
+        plt.show()
@@ -0,0 +1,175 @@
+import os
+import random
+import numpy as np
+
+
+class GAILMem(object):
+    def __init__(self):
+        self.states = []
+        self.actorProbs = []
+        self.actions = []
+        self.rewards = []
+        self.dones = []
+        self.memNum = 0
+        print("√√√√√Buffer Initialized Success√√√√√")
+
+    def clearMem(self):
+        """clearMemories"""
+        self.states = []
+        self.actorProbs = []
+        self.actions = []
+        self.rewards = []
+        self.dones = []
+        self.memNum = 0
+
+    def saveMemtoFile(self, dir: str):
+        """save memories ndarray to npz file
+
+        Args:
+            dir (str): save direction,like"GAIL-Expert-Data/",end with "/"
+        """
+        statesNP = np.asarray(self.states)
+        actorProbsNP = np.asarray(self.actorProbs)
+        actionsNP = np.asarray(self.actions)
+        rewardsNP = np.asarray(self.rewards)
+        donesNP = np.asarray(self.dones)
+        thisSaveDir = dir + "pack-" + str(self.memNum)
+        try:
+            np.savez(
+                thisSaveDir,
+                states=statesNP,
+                actorProbs=actorProbsNP,
+                actions=actionsNP,
+                rewards=rewardsNP,
+                dones=donesNP,
+            )
+        except FileNotFoundError:
+            os.mkdir(dir)
+            np.savez(
+                thisSaveDir,
+                states=statesNP,
+                actorProbs=actorProbsNP,
+                actions=actionsNP,
+                rewards=rewardsNP,
+                dones=donesNP,
+            )
+
+    def loadMemFile(self, dir: str):
+        """load memories from mpz file
+
+        Args:
+            dir (str): file direction
+        """
+        self.clearMem()
+        memFile = np.load(dir, allow_pickle=True)
+        self.states = memFile["states"].tolist()
+        self.actorProbs = memFile["actorProbs"].tolist()
+        self.actions = memFile["actions"].tolist()
+        self.rewards = memFile["rewards"].tolist()
+        self.dones = memFile["dones"].tolist()
+        self.memNum = len(self.states)
+
+    def getRandomSample(self, sampleNum: int = 0):
+        """get random unique sample set.
+
+        Args:
+            sampleNum (int, optional): sample number, while 0 return all samples. Defaults to 0.
+
+        Returns:
+            tuple: (states,actorProbs,actions,rewards,dones)
+        """
+        if sampleNum == 0:
+            return (
+                self.getStates(),
+                self.getActorProbs(),
+                self.getActions(),
+                self.getRewards(),
+                self.getDones(),
+            )
+        else:
+            randIndex = random.sample(range(0, self.memNum), sampleNum)
+            return (
+                self.standDims(np.asarray(self.states)[randIndex]),
+                self.standDims(np.asarray(self.actorProbs)[randIndex]),
+                self.standDims(np.asarray(self.actions)[randIndex]),
+                self.standDims(np.asarray(self.rewards)[randIndex]),
+                self.standDims(np.asarray(self.dones)[randIndex]),
+            )
+
+    def getStates(self):
+        """get all States data as ndarray
+
+        Returns:
+            ndarray: ndarray type State data
+        """
+        return self.standDims(np.asarray(self.states))
+
+    def getActorProbs(self):
+        """get all ActorProbs data as ndarray
+
+        Returns:
+            ndarray: ndarray type ActorProbs data
+        """
+
+        return self.standDims(np.asarray(self.actorProbs))
+
+    def getActions(self):
+        """get all Actions data as ndarray
+
+        Returns:
+            ndarray: ndarray type Actions data
+        """
+
+        return self.standDims(np.asarray(self.actions))
+
+    def getRewards(self):
+        """get all Rewards data as ndarray
+
+        Returns:
+            ndarray: ndarray type Rewards data
+        """
+
+        return self.standDims(np.asarray(self.rewards))
+
+    def getDones(self):
+        """get all Dones data as ndarray
+
+        Returns:
+            ndarray: ndarray type Dones data
+        """
+
+        return self.standDims(np.asarray(self.dones))
+
+    def standDims(self, data):
+        """standalize data's dimension
+
+        Args:
+            data (list): data list
+
+        Returns:
+            ndarray: ndarra type data
+        """
+        # standarlize data's dimension
+        if np.ndim(data) > 2:
+            return np.squeeze(data, axis=1)
+        elif np.ndim(data) < 2:
+            return np.expand_dims(data, axis=1)
+        else:
+            return np.asarray(data)
+
+    def saveMems(self, state, actorProb, action, reward, done):
+        """save memories
+
+        Args:
+            state (_type_): sates
+            actorProb (_type_): actor predict result
+            action (_type_): actor choosed action
+            reward (_type_): reward
+            done (function): done
+        """
+        self.states.append(state)
+        self.actorProbs.append(actorProb)
+        self.actions.append(action)
+        self.rewards.append(reward)
+        self.dones.append(done)
+        self.memNum += 1
@@ -0,0 +1,58 @@
+import keyboard
+import mouse
+import math
+
+
+class HumanActions:
+    def __init__(self, mouseDiscount: float = 10, screenW: int = 1920, screenH: int = 1080):
+        def multiPressed():
+            pass
+
+        keyboard.add_hotkey("w+a", multiPressed)
+        keyboard.add_hotkey("w+d", multiPressed)
+        keyboard.add_hotkey("s+a", multiPressed)
+        keyboard.add_hotkey("s+d", multiPressed)
+        self.screenW = screenW
+        self.screenH = screenH
+        self.MOUSEDISCOUNT = mouseDiscount
+        self.mouseSmooth = 5
+        self.mouseMax = 10
+
+    def getHumanActions(self):
+        x, _ = mouse.get_position()
+        xMovement = (x - self.screenW / 2) / self.MOUSEDISCOUNT
+        xMovement = self.smoothMouseMovement(xMovement)
+        ws = 0
+        ad = 0
+        click = 0
+        if keyboard.is_pressed("w"):
+            ws = 1
+        elif keyboard.is_pressed("s"):
+            ws = 2
+        if keyboard.is_pressed("d"):
+            ad = 1
+        elif keyboard.is_pressed("a"):
+            ad = 2
+        if keyboard.is_pressed("w+d"):
+            ws = 1
+            ad = 1
+        elif keyboard.is_pressed("w+a"):
+            ws = 1
+            ad = 2
+        elif keyboard.is_pressed("s+d"):
+            ws = 2
+            ad = 1
+        elif keyboard.is_pressed("s+a"):
+            ws = 2
+            ad = 2
+        if keyboard.is_pressed("0"):
+            click = 1
+
+        actions = [ws, ad, click, xMovement]
+
+        mouse.move(self.screenW / 2, self.screenH / 2)
+        return actions
+
+    def smoothMouseMovement(self, x: float):
+        out = (1 / (1 + math.exp(-x / self.mouseSmooth)) - 1 / 2) * self.mouseMax * 2
+        return out
@@ -0,0 +1,723 @@
+from os import mkdir
+import tensorflow as tf
+from tensorflow.python.ops.numpy_ops import ndarray
+import tensorflow_probability as tfp
+import numpy as np
+import time
+import math
+
+import datetime
+from PPOConfig import PPOConfig
+
+from tensorflow import keras
+from tensorflow.keras import layers
+from tensorflow.keras import optimizers
+
+EPS = 1e-10
+
+
+class PPO(object):
+    def __init__(
+        self,
+        stateSize: int,
+        disActShape: list,
+        conActSize: int,
+        conActRange: float,
+        PPOConfig: PPOConfig,
+    ):
+        """initialize PPO
+
+        Args:
+            stateSize (int): enviroment state size
+            disActShape (numpy): discrete Action shape.
+                                just like [3,2],means 2 type of dis actions,each act include 3 and 2 types
+                                if no discrete action output then use [0].
+            conActSize (int): continuous Action Size. if no continuous action output then use 0.
+            conActRange (float): continuous action range. -conActRange to +conActRange
+            PPOConfig (PPOConfig): PPO configuration
+        """
+        # check use dis action or not.
+        if disActShape == [0]:
+            # non dis action output
+            self.disActSize = 0
+            self.disOutputSize = 0
+        else:
+            # make sure disActShape greater than 1
+            try:
+                if np.any(np.array(disActShape) <= 1):
+                    raise ValueError(
+                        "disActShape error,disActShape should greater than 1 but get", disActShape
+                    )
+            except ValueError:
+                raise
+            self.disActSize = len(disActShape)
+            self.disOutputSize = sum(disActShape)
+
+        self.stateSize = stateSize
+        self.disActShape = disActShape
+        self.conActSize = conActSize
+        self.conActRange = conActRange
+        self.muSigSize = 2
+        self.conOutputSize = conActSize * self.muSigSize
+
+        # config
+        self.NNShape = PPOConfig.NNShape
+        self.criticLR = PPOConfig.criticLR
+        self.actorLR = PPOConfig.actorLR
+        self.gamma = PPOConfig.gamma
+        self.lmbda = PPOConfig.lmbda
+        self.clipRange = PPOConfig.clipRange
+        self.entropyWeight = PPOConfig.entropyWeight
+        self.trainEpochs = PPOConfig.trainEpochs
+        self.saveDir = PPOConfig.saveDir
+        self.loadModelDir = PPOConfig.loadModelDir
+        print("---------thisPPO Params---------")
+        print("self.stateSize = ", self.stateSize)
+        print("self.disActShape = ", self.disActShape)
+        print("self.disActSize", self.disActSize)
+        print("self.disOutputSize", self.disOutputSize)
+        print("self.conActSize = ", self.conActSize)
+        print("self.conActRange = ", self.conActRange)
+        print("self.conOutputSize = ", self.conOutputSize)
+
+        # config
+        print("---------thisPPO config---------")
+        print("self.NNShape = ", self.NNShape)
+        print("self.criticLR = ", self.criticLR)
+        print("self.actorLR = ", self.actorLR)
+        print("self.gamma = ", self.gamma)
+        print("self.lmbda = ", self.lmbda)
+        print("self.clipRange = ", self.clipRange)
+        print("self.entropyWeight = ", self.entropyWeight)
+        print("self.trainEpochs = ", self.trainEpochs)
+        print("self.saveDir = ", self.saveDir)
+        print("self.loadModelDir = ", self.loadModelDir)
+
+        # load NN or not
+        if self.loadModelDir is None:
+            # critc NN
+            self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
+            # actor NN
+            self.actor = self.buildActorNet(self.stateSize, compileModel=True)
+            print("---------Actor Model Create Success---------")
+            self.actor.summary()
+            print("---------Critic Model Create Success---------")
+            self.critic.summary()
+        else:
+            # critc NN
+            self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
+            # actor NN
+            self.actor = self.buildActorNet(self.stateSize, compileModel=True)
+            # load weight to Critic&Actor NN
+            self.loadWeightToModels(self.loadModelDir)
+            print("---------Actor Model Load Success---------")
+            self.actor.summary()
+            print("---------Critic Model Load Success---------")
+            self.critic.summary()
+
+    # Build Net
+    def buildActorNet(self, inputSize: int, compileModel: bool):
+        """build Actor Nueral Net and compile.Output:[disAct1,disAct2,disAct3,mu,sigma]
+
+        Args:
+            inputSize (int): InputLayer Nueral size.
+            compileModel (bool): compile Model or not.
+
+        Returns:
+            keras.Model: return Actor NN
+        """
+        # -----------Input Layers-----------
+        stateInput = layers.Input(shape=(inputSize,), name="stateInput")
+
+        # -------Intermediate layers--------
+        interLayers = []
+        interLayersIndex = 0
+        for neuralUnit in self.NNShape:
+            thisLayerName = "dense" + str(interLayersIndex)
+            if interLayersIndex == 0:
+                interLayers.append(
+                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
+                )
+            else:
+                interLayers.append(
+                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
+                )
+            interLayersIndex += 1
+
+        # ----------Output Layers-----------
+        outputLayersList = []
+        if self.disActSize != 0:
+            # while NN have discrete action output.
+            disActIndex = 0
+            for thisDisActDepth in self.disActShape:
+                thisDisActName = "disAct" + str(disActIndex)
+                outputLayersList.append(
+                    layers.Dense(thisDisActDepth, activation="softmax", name=thisDisActName)(
+                        interLayers[-1]
+                    )
+                )
+                disActIndex += 1
+        if self.conActSize != 0:
+            # while NN have continuous action output.
+            mu = tf.multiply(
+                layers.Dense(1, activation="tanh", name="muOut")(interLayers[-1]), self.conActRange
+            )  # mu，既正态分布位置参数
+            sigma = tf.add(
+                layers.Dense(1, activation="softplus", name="sigmaOut")(interLayers[-1]), EPS
+            )  # sigma，既正态分布尺度参数
+            outputLayersList.append(mu)
+            outputLayersList.append(sigma)
+        totalOut = layers.concatenate(outputLayersList, name="totalOut")  # package
+
+        # ----------Model Compile-----------
+        model = keras.Model(inputs=stateInput, outputs=totalOut)
+        if compileModel:  # Compile Model
+            actorOPT = optimizers.Adam(learning_rate=self.actorLR)
+            model.compile(optimizer=actorOPT, loss=self.aLoss())
+        return model
+
+    def buildCriticNet(self, inputSize: int, outputSize: int, compileModel: bool):
+        """build Critic Nueral Net and compile.Output:[Q]
+
+        Args:
+            inputSize (int): input size
+            outputSize (int): output size
+            compileModel (bool): compile Model or not.
+
+        Returns:
+            keras.Model: return Critic NN
+        """
+        # -----------Input Layers-----------
+        stateInput = keras.Input(shape=(inputSize,), name="stateInput")
+
+        # -------Intermediate layers--------
+        interLayers = []
+        interLayersIndex = 0
+        for neuralUnit in self.NNShape:
+            thisLayerName = "dense" + str(interLayersIndex)
+            if interLayersIndex == 0:
+                interLayers.append(
+                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
+                )
+            else:
+                interLayers.append(
+                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
+                )
+            interLayersIndex += 1
+
+        # ----------Output Layers-----------
+        output = layers.Dense(outputSize, activation=None)(interLayers[-1])
+
+        # ----------Model Compile-----------
+        model = keras.Model(inputs=stateInput, outputs=output)
+        if compileModel:
+            criticOPT = optimizers.Adam(learning_rate=self.criticLR)
+            model.compile(optimizer=criticOPT, loss=self.cLoss())
+        return model
+
+    # loss Function
+    # critic loss
+    def cLoss(self):
+        """Critic Loss function"""
+
+        def loss(y_true, y_pred):
+            # y_true: discountedR
+            # y_pred: critcV = model.predict(states)
+
+            adv = y_true - y_pred  # TD error
+            loss = tf.reduce_mean(tf.square(adv))
+            return loss
+
+        return loss
+
+    # actor loss
+    def aLoss(self):
+        """Actor Loss function"""
+
+        def getDiscreteALoss(nowProbs, oldProbs, disOneHotAct, actShape, advantage):
+            """get Discrete Action Loss
+
+            Args:
+                nowProbs (tf.constant): (length,actionProbSize)
+                oldProbs (tf.constant): (length,actionProbSize)
+                advantage (tf.constant): (length,)
+
+            Returns:
+                tf.constant: (length,)
+            """
+            entropy = tf.negative(
+                tf.reduce_mean(tf.math.multiply(nowProbs, tf.math.log(nowProbs + EPS)))
+            )
+            nowSingleProbs = tf.reduce_mean(tf.multiply(nowProbs, disOneHotAct), axis=1)
+            nowSingleProbs = tf.multiply(nowSingleProbs, actShape)
+            oldSingleProbs = tf.reduce_mean(tf.multiply(oldProbs, disOneHotAct), axis=1)
+            oldSingleProbs = tf.multiply(oldSingleProbs, actShape)
+            ratio = tf.math.divide(nowSingleProbs, oldSingleProbs + EPS)
+            value = tf.math.multiply(ratio, advantage)
+            clipRatio = tf.clip_by_value(ratio, 1.0 - self.clipRange, 1.0 + self.clipRange)
+            clipValue = tf.math.multiply(clipRatio, advantage)
+            loss = tf.math.negative(
+                tf.reduce_mean(tf.math.minimum(value, clipValue))
+                - tf.multiply(self.entropyWeight, entropy)
+            )
+            return loss
+
+        def getContinuousALoss(musig, actions, oldProbs, advantage):
+            """get Continuous Action Loss
+
+            Args:
+                musig (tf.constant): (length,2)
+                actions (tf.constant): (length,)
+                oldProbs (tf.constant): (length,)
+                advantage (tf.constant): (length,)
+
+            Returns:
+                tf.constant: (length,)
+            """
+            mu = musig[:, 0]
+            sigma = musig[:, 1]
+            dist = tfp.distributions.Normal(mu, sigma)
+
+            nowProbs = dist.prob(actions)
+            entropy = tf.reduce_mean(dist.entropy())
+
+            ratio = tf.math.divide(nowProbs, oldProbs + EPS)
+            value = tf.math.multiply(ratio, advantage)
+            clipRatio = tf.clip_by_value(ratio, 1.0 - self.clipRange, 1.0 + self.clipRange)
+            clipValue = tf.math.multiply(clipRatio, advantage)
+            loss = tf.negative(
+                tf.reduce_mean(tf.math.minimum(value, clipValue))
+                - tf.multiply(self.entropyWeight, entropy)
+            )
+            return loss
+
+        def loss(y_true, y_pred):
+            # y_true: [[disActProb..., conActProbs..., disOneHotActs..., conAct..., advantage]]
+            # y_pred: [[disActProb..., mu, sigma...]]
+            totalALoss = 0
+            totalActionNum = 0
+            advantage = y_true[:, -1]
+
+            if self.disActSize != 0:
+                # while NN have discrete action output.
+                oldDisProbs = y_true[:, 0 : self.disOutputSize]
+                nowDisProbs = y_pred[:, 0 : self.disOutputSize]  # [disAct1, disAct2, disAct3]
+                disOneHotActs = y_true[
+                    :,
+                    self.disOutputSize
+                    + self.conActSize : self.disOutputSize
+                    + self.conActSize
+                    + self.disOutputSize,
+                ]
+                lastDisActShape = 0
+                for thisShape in self.disActShape:
+                    thisNowDisProbs = nowDisProbs[:, lastDisActShape : lastDisActShape + thisShape]
+                    thisOldDisProbs = oldDisProbs[:, lastDisActShape : lastDisActShape + thisShape]
+                    thisDisOneHotActs = disOneHotActs[
+                        :, lastDisActShape : lastDisActShape + thisShape
+                    ]
+                    discreteALoss = getDiscreteALoss(
+                        thisNowDisProbs, thisOldDisProbs, thisDisOneHotActs, thisShape, advantage
+                    )
+                    lastDisActShape += thisShape
+                    totalALoss += discreteALoss
+                    totalActionNum += 1.0
+            if self.conActSize != 0:
+                # while NN have continuous action output.
+                oldConProbs = y_true[:, self.disOutputSize : self.disOutputSize + self.conActSize]
+                conActions = y_true[
+                    :,
+                    self.disOutputSize
+                    + self.conActSize
+                    + self.disOutputSize : self.disOutputSize
+                    + self.conActSize
+                    + self.disOutputSize
+                    + self.conActSize,
+                ]
+                nowConMusigs = y_pred[:, self.disOutputSize :]  # [musig1,musig2]
+                lastConAct = 0
+                for conAct in range(self.conActSize):
+                    thisNowConMusig = nowConMusigs[:, lastConAct : lastConAct + self.muSigSize]
+                    thisOldConProb = tf.squeeze(oldConProbs[:, conAct : conAct + 1])
+                    thisConAction = conActions[:, conAct]
+                    continuousAloss = getContinuousALoss(
+                        thisNowConMusig, thisConAction, thisOldConProb, advantage
+                    )
+                    totalALoss += continuousAloss
+                    totalActionNum += 1.0
+                    lastConAct += self.muSigSize
+            loss = tf.divide(totalALoss, totalActionNum)
+            return loss
+
+        return loss
+
+    # get Actions&values
+    def chooseAction(self, state: ndarray):
+        """Agent choose action to take
+
+        Args:
+            state (ndarray): enviroment state
+
+        Returns:
+            np.array:
+                actions,
+                    actions list,1dims like [0,1,1.5]
+                predictResult,
+                    actor NN predict Result output
+        """
+        # let actor choose action,use the normal distribution
+        # state = np.expand_dims(state,0)
+
+        # check state dimension is [stateNum,statesize]
+        if state.ndim != 2:
+            stateNum = int(len(state) / self.stateSize)
+            state = state.reshape([stateNum, self.stateSize])
+        predictResult = self.actor(state)  # get predict result [[disAct1, disAct2, disAct3, musig]]
+        # print("predictResult",predictResult)
+        # predictResult = predictResult.numpy()
+        actions = []
+        if self.disActSize != 0:
+            # while NN have discrete action output.
+            lastDisActShape = 0
+            for shape in self.disActShape:
+                thisDisActProbs = predictResult[:, lastDisActShape : lastDisActShape + shape]
+                dist = tfp.distributions.Categorical(probs=thisDisActProbs, dtype=tf.float32)
+                action = int(dist.sample().numpy()[0])
+                # action = np.argmax(thisDisActProbs)
+                actions.append(action)
+                lastDisActShape += shape
+        if self.conActSize != 0:
+            # while NN have continuous action output.
+            lastConAct = 0
+            for actIndex in range(self.conActSize):
+                thisMu = predictResult[:, self.disOutputSize + lastConAct]
+                thisSig = predictResult[:, self.disOutputSize + lastConAct + 1]
+                if math.isnan(thisMu) or math.isnan(thisSig):
+                    # check mu or sigma is nan
+                    print("chooseAction:mu or sigma is nan")
+                    print(predictResult)
+                thisDist = np.random.normal(loc=thisMu, scale=thisSig)
+                actions.append(np.clip(thisDist, -self.conActRange, self.conActRange)[0])
+                lastConAct += 2
+        return actions, predictResult
+
+    def trainCritcActor(
+        self,
+        states: ndarray,
+        oldActorResult: ndarray,
+        actions: ndarray,
+        rewards: ndarray,
+        dones: ndarray,
+        nextState: ndarray,
+        epochs: int = None,
+    ):
+        """train critic&actor use PPO ways
+
+        Args:
+            states (ndarray): states
+            oldActorResult (ndarray): actor predict result
+            actions (ndarray): predicted actions include both discrete actions and continuous actions
+            rewards (ndarray): rewards from enviroment
+            dones (ndarray): dones from enviroment
+            nextState (ndarray): next state from enviroment
+            epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.
+
+        Returns:
+            tf.constant: criticLoss, actorLoss
+        """
+
+        if epochs == None:
+            epochs = self.trainEpochs
+        criticValues = self.getCriticV(state=states)
+        discountedR = self.discountReward(nextState, criticValues, dones, rewards)
+        advantage = self.getGAE(discountedR, criticValues)
+
+        criticLoss = self.trainCritic(states, discountedR, epochs)
+        actorLoss = self.trainActor(states, oldActorResult, actions, advantage, epochs)
+        # print("A_Loss:", actorLoss, "C_Loss:", criticLoss)
+        return actorLoss, criticLoss
+
+    def trainCritic(self, states: ndarray, discountedR: ndarray, epochs: int = None):
+        """critic NN trainning function
+
+        Args:
+            states (ndarray): states
+            discountedR (ndarray): discounted rewards
+            epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.
+
+        Returns:
+            tf.constant: all critic losses
+        """
+        if epochs == None:
+            epochs = self.trainEpochs
+        his = self.critic.fit(x=states, y=discountedR, epochs=epochs, verbose=0)
+        return his.history["loss"]
+
+    def trainActor(
+        self,
+        states: ndarray,
+        oldActorResult: ndarray,
+        actions: ndarray,
+        advantage: ndarray,
+        epochs: int = None,
+    ):
+        """actor NN trainning function
+
+        Args:
+            states (ndarray): states
+            oldActorResult (ndarray): actor predict results
+            actions (ndarray): acotor predict actions
+            advantage (ndarray): GAE advantage
+            epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.
+
+        Returns:
+            tf.constant: all actor losses
+        """
+        # Trian Actor
+        # states: Buffer States
+        # actions: Buffer Actions
+        # discountedR: Discounted Rewards
+        # Epochs: just Epochs
+        if epochs == None:
+            epochs = self.trainEpochs
+        actions = np.asarray(actions, dtype=np.float32)
+
+        disActions = actions[:, 0 : self.disActSize]
+        conActions = actions[:, self.disActSize :]
+        oldDisProbs = oldActorResult[:, 0 : self.disOutputSize]  # [disAct1, disAct2, disAct3]
+        oldConMusigs = oldActorResult[:, self.disOutputSize :]  # [musig1,musig2]
+        if self.disActSize != 0:
+            disOneHotActs = self.getOneHotActs(disActions)
+            if self.conActSize != 0:
+                # while NN have discrete6 & continuous actions output.
+                oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
+                # pack [oldDisProbs,oldPiProbs,conActions,advantage] as y_true
+                y_true = np.hstack((oldDisProbs, oldPiProbs, disOneHotActs, conActions, advantage))
+            else:
+                # while NN have only discrete actions output.
+                # pack [oldDisProbs,advantage] as y_true
+                y_true = np.hstack((oldDisProbs, disOneHotActs, advantage))
+        else:
+            if self.conActSize != 0:
+                # while NN have only continuous action output.
+                oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
+                # pack [oldPiProbs,conActions,advantage] as y_true
+                y_true = np.hstack((oldPiProbs, conActions, advantage))
+            else:
+                print("trainActor:disActSize & conActSize error")
+                time.sleep(999999)
+        # assembly Actions history
+        # train start
+        if np.any(tf.math.is_nan(y_true)):
+            print("y_true got nan")
+            print("y_true", y_true)
+        his = self.actor.fit(x=states, y=y_true, epochs=epochs, verbose=0)
+        if np.any(tf.math.is_nan(his.history["loss"])):
+            print("his.history['loss'] is nan!")
+            print(his.history["loss"])
+        return his.history["loss"]
+
+    def saveWeights(self, score: float):
+        """save now NN's Weight. Use "models.save_weights" method.
+        Save as "tf" format "ckpt" file.
+
+        Args:
+            score (float): now score
+        """
+        actor_save_dir = (
+            self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
+        )
+        critic_save_dir = (
+            self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
+        )
+        self.actor.save_weights(actor_save_dir, save_format="tf")
+        self.critic.save_weights(critic_save_dir, save_format="tf")
+        # create an empty file named  as score to recored score
+        score_dir = (
+            self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
+        )
+        try:
+            scorefile = open(score_dir, "w")
+        except FileNotFoundError:
+            mkdir(self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/")
+            scorefile = open(score_dir, "w")
+        scorefile.close()
+        print("PPO Model's Weights Saved")
+
+    def loadWeightToModels(self, loadDir: str):
+        """load NN Model. Use "models.load_weights()" method.
+        Load "tf" format "ckpt" file.
+
+        Args:
+            loadDir (str): Model dir
+        """
+        actorDir = loadDir + "/actor/" + "actor.ckpt"
+        criticDir = loadDir + "/critic/" + "critic.ckpt"
+        self.actor.load_weights(actorDir)
+        self.critic.load_weights(criticDir)
+
+        print("++++++++++++++++++++++++++++++++++++")
+        print("++++++++++++Model Loaded++++++++++++")
+        print(loadDir)
+        print("++++++++++++++++++++++++++++++++++++")
+
+    def getCriticV(self, state: ndarray):
+        """get Critic predict V value
+
+        Args:
+            state (ndarray): Env state
+
+        Returns:
+            tensor: retrun Critic predict result
+        """
+        # if state.ndim < 2:
+        #    state = np.expand_dims(state,0)
+        if state.ndim != 2:
+            stateNum = int(len(state) / self.stateSize)
+            state = state.reshape([stateNum, self.stateSize])
+        return self.critic.predict(state)
+
+    def discountReward(self, nextState: ndarray, values: ndarray, dones: ndarray, rewards: ndarray):
+        """Discount future rewards
+
+        Args:
+            nextState (ndarray): next Env state
+            values (ndarray): critic predict values
+            dones (ndarray): dones from enviroment
+            rewards (ndarray): reward list of this episode
+
+        Returns:
+            ndarray: discounted rewards list,same shape as rewards that input
+        """
+        """
+        nextV = self.getCriticV(nextState)
+        dones = 1 - dones
+        discountedRewards = []
+        for i in reversed(range(len(rewards))):
+            nextV = rewards[i] + dones[i] * self.gamma * nextV
+            discountedRewards.append(nextV)
+        discountedRewards.reverse()  # reverse
+        discountedRewards = np.squeeze(discountedRewards)
+        discountedRewards = np.expand_dims(discountedRewards, axis=1)
+        # discountedRewards = np.array(discountedRewards)[:, np.newaxis]
+        return discountedRewards
+        """
+        """
+        nextV = self.getCriticV(nextState)
+        discountedRewards = []
+        for r in rewards[::-1]:
+            nextV = r + self.gamma * nextV
+            discountedRewards.append(nextV)
+        discountedRewards.reverse()  # reverse
+        discountedRewards = np.squeeze(discountedRewards)
+        discountedRewards = np.expand_dims(discountedRewards, axis=1)
+        # discountedRewards = np.array(discountedRewards)[:, np.newaxis]
+        print(discountedRewards)
+        return discountedRewards
+        """
+        g = 0
+        discountedRewards = []
+        lastValue = self.getCriticV(nextState)
+        values = np.append(values, lastValue, axis=0)
+        dones = 1 - dones
+        for i in reversed(range(len(rewards))):
+            delta = rewards[i] + self.gamma * values[i + 1] * dones[i] - values[i]
+            g = delta + self.gamma * self.lmbda * dones[i] * g
+            discountedRewards.append(g + values[i])
+        discountedRewards.reverse()
+        return np.asarray(discountedRewards)
+
+    def getGAE(self, discountedRewards: ndarray, values: ndarray):
+        """compute GAE adcantage
+
+        Args:
+            discountedRewards (ndarray): discounted rewards
+            values (ndarray): critic predict values
+
+        Returns:
+            ndarray: GAE advantage
+        """
+        advantage = discountedRewards - values
+        advantage = (advantage - np.mean(advantage)) / (np.std(advantage) + EPS)
+        return advantage
+
+    def conProb(self, mu: ndarray, sig: ndarray, x: ndarray):
+        """calculate probability when x in Normal distribution(mu,sigma)
+
+        Args:
+            mu (ndarray): mu
+            sig (ndarray): sigma
+            x (ndarray): x
+
+        Returns:
+            ndarray: probability
+        """
+        # 获取在正态分布mu,sig下当取x值时的概率
+        # return shape : (length,1)
+        mu = np.reshape(mu, (np.size(mu),))
+        sig = np.reshape(sig, (np.size(sig),))
+        x = np.reshape(x, (np.size(x),))
+
+        dist = tfp.distributions.Normal(mu, sig)
+        prob = dist.prob(x)
+
+        prob = np.reshape(prob, (np.size(x), 1))
+        # dist = 1./(tf.sqrt(2.*np.pi)*sig)
+        # prob = dist*tf.exp(-tf.square(x-mu)/(2.*tf.square(sig)))
+        return prob
+
+    def getOneHotActs(self, disActions):
+        """one hot action encoder
+
+        Args:
+            disActions (ndarray): discrete actions
+
+        Returns:
+            ndarray: one hot actions
+        """
+        actIndex = 0
+        for thisShape in self.disActShape:
+            thisActs = disActions[:, actIndex]
+            thisOneHotAct = tf.squeeze(tf.one_hot(thisActs, thisShape)).numpy()
+            if actIndex == 0:
+                oneHotActs = thisOneHotAct
+            else:
+                oneHotActs = np.append(oneHotActs, thisOneHotAct, axis=1)
+            actIndex += 1
+        return oneHotActs
+
+    def getAverageEntropy(self, probs: ndarray):
+        """get average dis&con ACT Entropys
+
+        Args:
+            probs (ndarray): actor NN predict result
+
+        Returns:
+            float: average total entropy
+            list: discrete entropys
+            list: continuous entropys
+        """
+        discreteEntropys = []
+        continuousEntropys = []
+        if self.disActSize != 0:
+            disProbs = probs[:, 0 : self.disOutputSize]
+            lastDisActIndex = 0
+            for actShape in self.disActShape:
+                thisDisProbs = disProbs[:, lastDisActIndex : lastDisActIndex + actShape]
+                lastDisActIndex += actShape
+                discreteEntropys.append(
+                    tf.negative(
+                        tf.reduce_mean(
+                            tf.math.multiply(thisDisProbs, tf.math.log(thisDisProbs + EPS))
+                        )
+                    )
+                )
+        if self.conActSize != 0:
+            conProbs = probs[:, self.disOutputSize :]
+            conActIndex = 0
+            for i in range(self.conActSize):
+                thisConProbs = conProbs[:, conActIndex : conActIndex + 2]
+                conActIndex += 2
+                continuousEntropys.append(tf.reduce_mean(thisConProbs[:, 1]))
+        averageEntropy = np.mean([np.mean(discreteEntropys), np.mean(continuousEntropys)])
+        return averageEntropy, discreteEntropys, continuousEntropys
@@ -0,0 +1,65 @@
+import numpy as np
+
+
+class PPOBuffer(object):
+    def __init__(self):
+        self.states = []
+        self.actorProbs = []
+        self.actions = []
+        self.rewards = []
+        self.dones = []
+        print("√√√√√Buffer Initialized Success√√√√√")
+
+    def clearBuffer(self):
+        self.states = []
+        self.actorProbs = []
+        self.actions = []
+        self.rewards = []
+        self.dones = []
+
+    def getStates(self):
+        return self.standDims(np.asarray(self.states))
+
+    def getActorProbs(self):
+        return self.standDims(np.asarray(self.actorProbs))
+
+    def getActions(self):
+        return self.standDims(np.asarray(self.actions))
+
+    def getRewards(self):
+        return self.standDims(np.asarray(self.rewards))
+
+    def getDones(self):
+        return self.standDims(np.asarray(self.dones))
+
+    def saveState(self, state):
+        self.states.append(state)
+
+    def saveAction(self, action):
+        self.actions.append(action)
+
+    def saveReward(self, reward):
+        self.rewards.append(reward)
+
+    def standDims(self, data):
+        # standarlize data's dimension
+        if np.ndim(data) > 2:
+            return np.squeeze(data, axis=1)
+        elif np.ndim(data) < 2:
+            return np.expand_dims(data, axis=1)
+        else:
+            return np.asarray(data)
+
+    def saveBuffers(self, state, actorProb, action, reward, done):
+        self.states.append(state)
+        self.actorProbs.append(actorProb)
+        self.actions.append(action)
+        self.rewards.append(reward)
+        self.dones.append(done)
+        """
+        print("self.states", self.states)
+        print("self.actions", self.actions)
+        print("self.rewards", self.rewards)
+        print("self.dones", self.dones)
+        print("self.values", self.values)
+        """
@@ -0,0 +1,15 @@
+import datetime
+from typing import NamedTuple, Optional
+
+
+class PPOConfig(NamedTuple):
+    NNShape: list = [256, 256, 128]
+    actorLR: float = 2e-3  # Actor Net Learning
+    criticLR: float = 2e-3  # Critic Net Learning
+    gamma: float = 0.99
+    lmbda: float = 0.95
+    clipRange: float = 0.20
+    entropyWeight: float = 1e-2
+    trainEpochs: int = 8
+    saveDir: str = "PPO-Model/" + datetime.datetime.now().strftime("%m%d-%H%M") + "/"
+    loadModelDir: Optional[str] = None
@@ -0,0 +1,37 @@
+import matplotlib.pyplot as plt
+
+
+darkBlue = "#011627"
+darkWhite = "#c9d2df"
+
+
+class PPOHistory(object):
+    def __init__(self):
+        self.meanRewards = []
+        self.entropys = []
+        self.actorLosses = []
+        self.criticLosses = []
+
+    def saveHis(self, rewards, entropys, aLosses, cLosses):
+        self.meanRewards.extend([rewards])
+        self.entropys.extend([entropys])
+        self.actorLosses.extend(aLosses)
+        self.criticLosses.extend(cLosses)
+
+    def drawHis(self):
+        def setSubFig(subFig, data, title):
+            subFig.set_facecolor(darkBlue)
+            subFig.tick_params(colors=darkWhite)
+            subFig.spines["top"].set_color(darkWhite)
+            subFig.spines["bottom"].set_color(darkWhite)
+            subFig.spines["left"].set_color(darkWhite)
+            subFig.spines["right"].set_color(darkWhite)
+            subFig.plot(range(len(data)), data, color=darkWhite, label=title)
+            subFig.set_title(title, color=darkWhite)
+
+        fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(21, 13), facecolor=darkBlue)
+        setSubFig(ax1, self.meanRewards, "meanRewards")
+        setSubFig(ax2, self.entropys, "entropys")
+        setSubFig(ax3, self.actorLosses, "actorLosses")
+        setSubFig(ax4, self.criticLosses, "criticLosses")
+        plt.show()
@@ -0,0 +1,161 @@
+from mlagents_envs.base_env import ActionTuple
+from mlagents_envs.environment import UnityEnvironment
+
+import numpy as np
+from numpy import ndarray
+
+
+class makeEnv(object):
+    def __init__(
+        self,
+        envPath: str,
+        workerID: int = 1,
+        basePort: int = 100,
+        stackSize: int = 1,
+        stackIntercal: int = 0,
+    ):
+        self.env = UnityEnvironment(
+            file_name=envPath,
+            seed=1,
+            side_channels=[],
+            worker_id=workerID,
+            base_port=basePort,
+        )
+        self.env.reset()
+
+        # get enviroment specs
+        self.LOAD_DIR_SIZE_IN_STATE = 3
+        self.TRACKED_AGENT = -1
+        self.BEHA_SPECS = self.env.behavior_specs
+        self.BEHA_NAME = list(self.BEHA_SPECS)[0]
+        self.SPEC = self.BEHA_SPECS[self.BEHA_NAME]
+        self.OBSERVATION_SPECS = self.SPEC.observation_specs[0]  # observation spec
+        self.ACTION_SPEC = self.SPEC.action_spec  # action specs
+
+        self.DISCRETE_SIZE = self.ACTION_SPEC.discrete_size
+        self.DISCRETE_SHAPE = list(self.ACTION_SPEC.discrete_branches)
+        self.CONTINUOUS_SIZE = self.ACTION_SPEC.continuous_size
+        self.SINGLE_STATE_SIZE = self.OBSERVATION_SPECS.shape[0] - self.LOAD_DIR_SIZE_IN_STATE
+        self.STATE_SIZE = self.SINGLE_STATE_SIZE * stackSize
+
+        # stacked State
+        self.STACK_SIZE = stackSize
+        self.STATE_BUFFER_SIZE = stackSize + ((stackSize - 1) * stackIntercal)
+        self.STACK_INDEX = list(range(0, self.STATE_BUFFER_SIZE, stackIntercal + 1))
+        self.statesBuffer = np.array([[0.0] * self.SINGLE_STATE_SIZE] * self.STATE_BUFFER_SIZE)
+        print("√√√√√Enviroment Initialized Success√√√√√")
+
+    def step(
+        self,
+        actions: list,
+        behaviorName: ndarray = None,
+        trackedAgent: int = None,
+    ):
+        """change ations list to ActionTuple then send it to enviroment
+
+        Args:
+            actions (list): PPO chooseAction output action list
+            behaviorName (ndarray, optional): behaviorName. Defaults to None.
+            trackedAgent (int, optional): trackedAgentID. Defaults to None.
+
+        Returns:
+            ndarray: nextState, reward, done, loadDir, saveNow
+        """
+        # take action to enviroment
+        # return mextState,reward,done
+        if self.DISCRETE_SIZE == 0:
+            # create empty discrete action
+            discreteActions = np.asarray([[0]])
+        else:
+            # create discrete action from actions list
+            discreteActions = np.asanyarray([actions[0 : self.DISCRETE_SIZE]])
+        if self.CONTINUOUS_SIZE == 0:
+            # create empty continuous action
+            continuousActions = np.asanyarray([[0.0]])
+        else:
+            # create continuous actions from actions list
+            continuousActions = np.asanyarray([actions[self.DISCRETE_SIZE :]])
+
+        if behaviorName is None:
+            behaviorName = self.BEHA_NAME
+        if trackedAgent is None:
+            trackedAgent = self.TRACKED_AGENT
+
+        # create actionTuple
+        thisActionTuple = ActionTuple(continuous=continuousActions, discrete=discreteActions)
+        # take action to env
+        self.env.set_actions(behavior_name=behaviorName, action=thisActionTuple)
+        self.env.step()
+        # get nextState & reward & done after this action
+        nextState, reward, done, loadDir, saveNow = self.getSteps(behaviorName, trackedAgent)
+        return nextState, reward, done, loadDir, saveNow
+
+    def getSteps(self, behaviorName=None, trackedAgent=None):
+        """get enviroment now observations.
+        Include State, Reward, Done, LoadDir, SaveNow
+
+        Args:
+            behaviorName (_type_, optional): behaviorName. Defaults to None.
+            trackedAgent (_type_, optional): trackedAgent. Defaults to None.
+
+        Returns:
+            ndarray: nextState, reward, done, loadDir, saveNow
+        """
+        # get nextState & reward & done
+        if behaviorName is None:
+            behaviorName = self.BEHA_NAME
+        decisionSteps, terminalSteps = self.env.get_steps(behaviorName)
+        if self.TRACKED_AGENT == -1 and len(decisionSteps) >= 1:
+            self.TRACKED_AGENT = decisionSteps.agent_id[0]
+        if trackedAgent is None:
+            trackedAgent = self.TRACKED_AGENT
+
+        if trackedAgent in decisionSteps:  # ゲーム終了していない場合、環境状態がdecision_stepsに保存される
+            nextState = decisionSteps[trackedAgent].obs[0]
+            nextState = np.reshape(
+                nextState, [self.SINGLE_STATE_SIZE + self.LOAD_DIR_SIZE_IN_STATE]
+            )
+            saveNow = nextState[-1]
+            loadDir = nextState[-3:-1]
+            nextState = nextState[:-3]
+            reward = decisionSteps[trackedAgent].reward
+            done = False
+        if trackedAgent in terminalSteps:  # ゲーム終了した場合、環境状態がterminal_stepsに保存される
+            nextState = terminalSteps[trackedAgent].obs[0]
+            nextState = np.reshape(
+                nextState, [self.SINGLE_STATE_SIZE + self.LOAD_DIR_SIZE_IN_STATE]
+            )
+            saveNow = nextState[-1]
+            loadDir = nextState[-3:-1]
+            nextState = nextState[:-3]
+            reward = terminalSteps[trackedAgent].reward
+            done = True
+
+        # stack state
+        stackedStates = self.stackStates(nextState)
+        return stackedStates, reward, done, loadDir, saveNow
+
+    def reset(self):
+        """reset enviroment and get observations
+
+        Returns:
+            ndarray: nextState, reward, done, loadDir, saveNow
+        """
+        # reset buffer
+        self.statesBuffer = np.array([[0.0] * self.SINGLE_STATE_SIZE] * self.STATE_BUFFER_SIZE)
+        # reset env
+        self.env.reset()
+        nextState, reward, done, loadDir, saveNow = self.getSteps()
+        return nextState, reward, done, loadDir, saveNow
+
+    def stackStates(self, state):
+        # save buffer
+        self.statesBuffer[0:-1] = self.statesBuffer[1:]
+        self.statesBuffer[-1] = state
+
+        # return stacked states
+        return np.reshape(self.statesBuffer[self.STACK_INDEX], (self.STATE_SIZE))
+
+    def render(self):
+        """render enviroment"""
+        self.env.render()
@@ -0,0 +1,160 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import env"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "31\n",
+      "5\n",
+      "3\n"
+     ]
+    }
+   ],
+   "source": [
+    "a=1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "包含None！\n",
+      "[123 None]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "def qa(aa = None):\n",
+    "    try:\n",
+    "        isaanone = aa.any() == None\n",
+    "        if aa.all() == None:\n",
+    "            print(\"包含None！\")\n",
+    "    except:\n",
+    "        isaanone =True\n",
+    "    if isaanone:\n",
+    "        print('none')\n",
+    "    else:\n",
+    "        print(aa)\n",
+    "\n",
+    "qa(np.array([123,None]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[[1 1 1 1 1]\n",
+      "  [1 2 1 1 1]]\n",
+      "\n",
+      " [[2 1 3 4 5]\n",
+      "  [2 2 3 4 5]]\n",
+      "\n",
+      " [[3 1 3 4 5]\n",
+      "  [3 2 3 4 5]]]\n",
+      "-\n",
+      "tf.Tensor(\n",
+      "[[1 1 1 1 1]\n",
+      " [2 1 3 4 5]\n",
+      " [3 1 3 4 5]], shape=(3, 5), dtype=int32)\n",
+      "-\n",
+      "tf.Tensor(\n",
+      "[[1 2 1 1 1]\n",
+      " [2 2 3 4 5]\n",
+      " [3 2 3 4 5]], shape=(3, 5), dtype=int32)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "\n",
+    "aa = np.array([[[1,1,1,1,1],[1,2,1,1,1],[1,3,1,1,1]],\n",
+    "               [[2,1,3,4,5],[2,2,3,4,5],[2,3,3,4,5]],\n",
+    "               [[3,1,3,4,5],[3,2,3,4,5],[3,3,3,4,5]]])\n",
+    "tt = tf.constant(aa)\n",
+    "bb = np.array([6,3,6,3,2,3])\n",
+    "\n",
+    "print(aa[:,0:2])\n",
+    "aa[:,2:]\n",
+    "\n",
+    "for asd in tf.transpose(aa[:,0:2],perm=[1,0,2]):\n",
+    "    print('-')\n",
+    "    print(asd)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tf.Tensor: shape=(), dtype=int32, numpy=1>"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import tensorflow as tf\n",
+    "aa = tf.constant(0)\n",
+    "bb = aa+1\n",
+    "bb\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.9.7 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
@@ -0,0 +1,57 @@
+import numpy as np
+
+
+class rewardAI(object):
+    def __init__(
+        self,
+        nonReward,
+        shootReward,
+        shootWithoutReadyReward,
+        hitReward,
+        winReward,
+        loseReward,
+        killReward,
+    ):
+        self.nonReward = nonReward
+        self.shootReward = shootReward
+        self.shootWithoutReadyReward = shootWithoutReadyReward
+        self.hitReward = hitReward
+        self.winReward = winReward
+        self.loseReward = loseReward
+        self.killReward = killReward
+
+    def getRewards(self, remainTime):
+        nonR = self.getnonReward()
+        shootR = self.getshootReward()
+        shootWithoutReadyR = self.getshootWithoutReadyReward()
+        hitR = self.gethitReward()
+        winR = self.getwinReward(remainTime)
+        loseR = self.getloseReward()
+        killR = self.getkillReward(remainTime)
+
+        rewards = np.array(
+            [[nonR, shootR, shootWithoutReadyR, hitR, winR, loseR, killR]],
+            dtype=np.float,
+        )
+        return rewards
+
+    def getnonReward(self):
+        return self.nonReward
+
+    def getshootReward(self):
+        return self.shootReward
+
+    def getshootWithoutReadyReward(self):
+        return self.shootWithoutReadyReward
+
+    def gethitReward(self):
+        return self.hitReward
+
+    def getwinReward(self, time):
+        return self.winReward + time
+
+    def getloseReward(self):
+        return self.loseReward
+
+    def getkillReward(self, time):
+        return self.killReward + time
@@ -0,0 +1,61 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mlagents_envs.environment import UnityEnvironment\n",
+    "from gym_unity.envs import UnityToGymWrapper\n",
+    "\n",
+    "ENV_PATH = \"../Build-CloseEnemyCut/Aimbot-PPO\"\n",
+    "WORKER_ID = 2\n",
+    "BASE_PORT = 2002\n",
+    "\n",
+    "def main():\n",
+    "    unity_env = UnityEnvironment(file_name=ENV_PATH,seed = 1,side_channels=[],worker_id = WORKER_ID,base_port=BASE_PORT)\n",
+    "    env = UnityToGymWrapper(unity_env,  uint8_visual=True,allow_multiple_obs=True)\n",
+    "    env.reset()\n",
+    "    for _ in range(1000):\n",
+    "        env.render()\n",
+    "        o,r,d,_ = env.step(env.action_space.sample()) #random action\n",
+    "        if d:\n",
+    "            env.reset()\n",
+    "    env.close()    \n",
+    "    \n",
+    "\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    main()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.7 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
@@ -0,0 +1,615 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[ 1,  2,  3,  7,  8,  9, 10]])"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "state = np.array([[1, 2, 3], [1, 2, 3]])\n",
+    "aaa = np.array([[123]])\n",
+    "\n",
+    "state[:, -1]\n",
+    "\n",
+    "np.append([[1, 2, 3]], [[7, 8, 9, 10]], axis=1)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "import numpy as np\n",
+    "\n",
+    "aa = tf.constant([[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]])\n",
+    "bb = tf.constant([1, 2, 3, 4])\n",
+    "\n",
+    "print(tf.expand_dims(bb, axis=1))\n",
+    "\n",
+    "cc = tf.math.multiply(aa, tf.expand_dims(bb, axis=1))\n",
+    "\n",
+    "print(cc)\n",
+    "print(tf.shape(aa))\n",
+    "\n",
+    "print(aa[:, 2:3])\n",
+    "\n",
+    "aa = tf.constant([1.0, 2.0, 3.0, np.nan])\n",
+    "print(np.any(tf.math.is_nan(aa)))\n",
+    "if np.any(tf.math.is_nan(aa)):\n",
+    "    print(\"true\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "\n",
+    "prob = tf.constant([0.3, 0.3, 0.0])\n",
+    "\n",
+    "entropy = tf.reduce_mean(\n",
+    "    tf.math.negative(tf.math.multiply(prob, tf.math.log(prob)))\n",
+    ")\n",
+    "\n",
+    "print(entropy)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "int 23\n",
+      "str twenty three\n",
+      "bool True\n",
+      "error\n"
+     ]
+    }
+   ],
+   "source": [
+    "from functools import singledispatchmethod\n",
+    "\n",
+    "\n",
+    "class person:\n",
+    "    @singledispatchmethod\n",
+    "    def age(self, arg):\n",
+    "        print(\"error\")\n",
+    "\n",
+    "    @age.register(int)\n",
+    "    def _(self, arg: int):\n",
+    "        print(\"int\", arg)\n",
+    "\n",
+    "    @age.register(str)\n",
+    "    def _(self, arg: str):\n",
+    "        print(\"str\", arg)\n",
+    "\n",
+    "    @age.register(bool)\n",
+    "    def _(self, arg: bool):\n",
+    "        print(\"bool\", arg)\n",
+    "\n",
+    "\n",
+    "p = person()\n",
+    "p.age(23)  # int\n",
+    "p.age(\"twenty three\")  # str\n",
+    "p.age(True)  # bool\n",
+    "p.age([\"23\"])  # list\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "import tensorflow.keras.layers as layers\n",
+    "\n",
+    "\n",
+    "def conv_bn_relu(inputs, chs, reps):\n",
+    "    x = inputs\n",
+    "    for i in range(reps):\n",
+    "        x = layers.Conv2D(chs, 3, padding=\"same\")(x)\n",
+    "        x = layers.BatchNormalization()(x)\n",
+    "        x = layers.ReLU()(x)\n",
+    "    return x\n",
+    "\n",
+    "\n",
+    "def create_model():\n",
+    "    inputs = layers.Input((32, 32, 3))\n",
+    "    x = conv_bn_relu(inputs, 64, 3)\n",
+    "    x = layers.AveragePooling2D(2)(x)\n",
+    "    x = conv_bn_relu(x, 128, 3)\n",
+    "    x = layers.AveragePooling2D(2)(x)\n",
+    "    x = conv_bn_relu(x, 256, 3)\n",
+    "    x = layers.GlobalAveragePooling2D()(x)\n",
+    "    x = layers.Dense(10, activation=\"softmax\")(x)\n",
+    "    return tf.keras.models.Model(inputs, x)\n",
+    "\n",
+    "\n",
+    "def perprocess(img, label):\n",
+    "    img = tf.cast(img, tf.float32) / 255.0\n",
+    "    label = tf.cast(label, tf.float32)\n",
+    "    return img, label\n",
+    "\n",
+    "\n",
+    "def train():\n",
+    "    (X_train, y_train), (\n",
+    "        X_test,\n",
+    "        y_test,\n",
+    "    ) = tf.keras.datasets.cifar10.load_data()\n",
+    "    trainset = (\n",
+    "        tf.data.Dataset.from_tensor_slices((X_train, y_train))\n",
+    "        .map(perprocess)\n",
+    "        .shuffle(4096)\n",
+    "        .batch(128)\n",
+    "        .repeat()\n",
+    "        .prefetch(50)\n",
+    "    )\n",
+    "\n",
+    "    model = create_model()\n",
+    "    model.compile(\"adam\", \"sparse_categorical_crossentropy\", [\"acc\"])\n",
+    "\n",
+    "    model.fit(trainset, steps_per_epoch=50000 // 128, epochs=1)\n",
+    "    # 'Adam/conv2d/kernel/m:0' shape=(3, 3, 3, 64)\n",
+    "    print(model.optimizer.weights[1][0, 0, 0, :10])\n",
+    "    # <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 3, 64)\n",
+    "    print(model.weights[0][0, 0, 0, :10])\n",
+    "\n",
+    "    model.save_weights(\"model_tf.ckpt\", save_format=\"tf\")  # デフォルト\n",
+    "    model.save_weights(\"model_h5.h5\", save_format=\"h5\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "390/390 [==============================] - 28s 59ms/step - loss: 1.2548 - acc: 0.5441\n",
+      "tf.Tensor(\n",
+      "[ 0.00190057 -0.00765918  0.00163367  0.00782851  0.02600338  0.00516749\n",
+      " -0.00424899  0.01562062 -0.0022073  -0.00355565], shape=(10,), dtype=float32)\n",
+      "tf.Tensor(\n",
+      "[ 0.07978954 -0.04595745 -0.03745254 -0.03701654  0.03296526 -0.11328737\n",
+      " -0.10719797  0.00874998  0.0226855   0.02288487], shape=(10,), dtype=float32)\n"
+     ]
+    }
+   ],
+   "source": [
+    "train()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tf.Tensor(\n",
+      "[ 0.07978954 -0.04595745 -0.03745254 -0.03701654  0.03296526 -0.11328737\n",
+      " -0.10719797  0.00874998  0.0226855   0.02288487], shape=(10,), dtype=float32)\n",
+      "tf.Tensor(\n",
+      "[ 0.00190057 -0.00765918  0.00163367  0.00782851  0.02600338  0.00516749\n",
+      " -0.00424899  0.01562062 -0.0022073  -0.00355565], shape=(10,), dtype=float32)\n"
+     ]
+    }
+   ],
+   "source": [
+    "def load_tf_w_zero_grad():\n",
+    "    model = create_model()\n",
+    "    model.compile(\"adam\", \"sparse_categorical_crossentropy\", [\"acc\"])\n",
+    "\n",
+    "    zero_grad = [tf.zeros_like(x) for x in model.weights]\n",
+    "    model.optimizer.apply_gradients(zip(zero_grad, model.weights))\n",
+    "\n",
+    "    model.load_weights(\"model_tf.ckpt\")\n",
+    "    # これでようやくオプティマイザーの値も同一になる\n",
+    "    print(model.weights[0][0, 0, 0, :10])\n",
+    "    print(model.optimizer.weights[1][0, 0, 0, :10])\n",
+    "\n",
+    "\n",
+    "load_tf_w_zero_grad()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[0.]])"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "a = np.array([10, 20, 30, 0])\n",
+    "\n",
+    "np.asarray([[0.]])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1.5"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "asd = [1,2,3,np.array([0.5]),np.array([0.5])]\n",
+    "\n",
+    "asd[3:]\n",
+    "len(asd)\n",
+    "\n",
+    "np.mean([1,2])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.0\n",
+      "0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import time\n",
+    "import pyautogui as pag\n",
+    "\n",
+    "from pynput.mouse import Button, Controller\n",
+    "\n",
+    "w = pag.size().width\n",
+    "h = pag.size().height\n",
+    "mouse = Controller()\n",
+    "\n",
+    "nowt = time.time()\n",
+    "\n",
+    "middletime = time.time() - nowt\n",
+    "print(middletime)\n",
+    "# print(nowPos-(w/2))\n",
+    "\n",
+    "print(time.time() - middletime - nowt)\n",
+    "while True:\n",
+    "    x,_ = mouse.position\n",
+    "    #print(mouse.press)\n",
+    "    #print(mouse.position)\n",
+    "    \n",
+    "    mouse.position = (w / 2, h / 2)\n",
+    "    time.sleep(1/60)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "import pyautogui as pag\n",
+    "\n",
+    "import mouse\n",
+    "\n",
+    "w = pag.size().width\n",
+    "h = pag.size().height\n",
+    "\n",
+    "nowt = time.time()\n",
+    "\n",
+    "middletime = time.time() - nowt\n",
+    "print(middletime)\n",
+    "# print(nowPos-(w/2))\n",
+    "\n",
+    "print(time.time() - middletime - nowt)\n",
+    "while True:\n",
+    "    x = mouse.get_position()\n",
+    "    print(x)\n",
+    "    #print(mouse.position)\n",
+    "    \n",
+    "    mouse.move(w / 2, h / 2)\n",
+    "    time.sleep(1/60)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "3\n",
+      "deque([[0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [1, 1, 1, 1, 1]], maxlen=3)\n",
+      "3\n",
+      "deque([[0.0, 0.0, 0.0, 0.0, 0.0], [1, 1, 1, 1, 1], [2, 2, 2, 2, 2]], maxlen=3)\n",
+      "3\n",
+      "deque([[1, 1, 1, 1, 1], [2, 2, 2, 2, 2], [3, 3, 3, 3, 3]], maxlen=3)\n",
+      "3\n",
+      "deque([[2, 2, 2, 2, 2], [3, 3, 3, 3, 3], [4, 4, 4, 4, 4]], maxlen=3)\n",
+      "3\n",
+      "deque([[3, 3, 3, 3, 3], [4, 4, 4, 4, 4], [5, 5, 5, 5, 5]], maxlen=3)\n",
+      "3\n",
+      "deque([[4, 4, 4, 4, 4], [5, 5, 5, 5, 5], [6, 6, 6, 6, 6]], maxlen=3)\n",
+      "3\n",
+      "deque([[5, 5, 5, 5, 5], [6, 6, 6, 6, 6], [7, 7, 7, 7, 7]], maxlen=3)\n",
+      "3\n",
+      "deque([[6, 6, 6, 6, 6], [7, 7, 7, 7, 7], [8, 8, 8, 8, 8]], maxlen=3)\n",
+      "3\n",
+      "deque([[7, 7, 7, 7, 7], [8, 8, 8, 8, 8], [9, 9, 9, 9, 9]], maxlen=3)\n"
+     ]
+    }
+   ],
+   "source": [
+    "from collections import deque\n",
+    "import numpy as np\n",
+    "\n",
+    "maxBuffer = 3\n",
+    "stateSize = 5\n",
+    "\n",
+    "aa = deque([[0.0]*stateSize],maxlen=maxBuffer)\n",
+    "\n",
+    "def ss(s):\n",
+    "    aa.append(s)\n",
+    "    if len(aa) < maxBuffer:\n",
+    "        for i in range(maxBuffer - len(aa)):\n",
+    "            aa.appendleft([0.0] * stateSize)\n",
+    "\n",
+    "for i in range(1,10):\n",
+    "    ss([i,i,i,i,i])\n",
+    "    print(len(aa))\n",
+    "    print(aa)\n",
+    "'''\n",
+    "3\n",
+    "deque([[0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [1, 1, 1, 1, 1]], maxlen=3)\n",
+    "3\n",
+    "deque([[0.0, 0.0, 0.0, 0.0, 0.0], [1, 1, 1, 1, 1], [2, 2, 2, 2, 2]], maxlen=3)\n",
+    "3\n",
+    "deque([[1, 1, 1, 1, 1], [2, 2, 2, 2, 2], [3, 3, 3, 3, 3]], maxlen=3)\n",
+    "3\n",
+    "deque([[2, 2, 2, 2, 2], [3, 3, 3, 3, 3], [4, 4, 4, 4, 4]], maxlen=3)\n",
+    "3\n",
+    "deque([[3, 3, 3, 3, 3], [4, 4, 4, 4, 4], [5, 5, 5, 5, 5]], maxlen=3)\n",
+    "3\n",
+    "deque([[4, 4, 4, 4, 4], [5, 5, 5, 5, 5], [6, 6, 6, 6, 6]], maxlen=3)\n",
+    "3\n",
+    "deque([[5, 5, 5, 5, 5], [6, 6, 6, 6, 6], [7, 7, 7, 7, 7]], maxlen=3)\n",
+    "3\n",
+    "deque([[6, 6, 6, 6, 6], [7, 7, 7, 7, 7], [8, 8, 8, 8, 8]], maxlen=3)\n",
+    "3\n",
+    "deque([[7, 7, 7, 7, 7], [8, 8, 8, 8, 8], [9, 9, 9, 9, 9]], maxlen=3)'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([0, 1, 2, 0, 1, 2, 0, 1, 2])"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from collections import deque\n",
+    "import numpy as np\n",
+    "\n",
+    "aa = np.array([range(0,3)]*5)\n",
+    "np.reshape(aa[[0,1,2]],(9))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'int'>\n",
+      "<class 'float'>\n",
+      "<class 'list'>\n",
+      "300\n",
+      "256.1\n",
+      "[300, 256.1]\n",
+      "300\n",
+      "256.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 変数を設定\n",
+    "ringo_int = 300\n",
+    "ringo_float = 256.1\n",
+    "ringo_list = [ringo_int, ringo_float]\n",
+    "\n",
+    "# 型を確認\n",
+    "print(type(ringo_int))\n",
+    "print(type(ringo_float))\n",
+    "print(type(ringo_list))\n",
+    "\n",
+    "# 値を表示\n",
+    "print(ringo_int)\n",
+    "print(ringo_float)\n",
+    "print(ringo_list)\n",
+    "\n",
+    "# 配列から要素を取り出す\n",
+    "print(ringo_list[0]) # ここでエラーになるという。どぼじで？？？\n",
+    "print(ringo_list[1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "dirrr = \"GAIL-Expert-Data/1014-1302/pack-24957.npz\"\n",
+    "\n",
+    "memFile = np.load(dirrr, allow_pickle=True)\n",
+    "states = memFile[\"states\"].tolist()\n",
+    "actorProbs = memFile[\"actorProbs\"].tolist()\n",
+    "actions = memFile[\"actions\"].tolist()\n",
+    "rewards = memFile[\"rewards\"].tolist()\n",
+    "dones = memFile[\"dones\"].tolist()\n",
+    "memNum = len(states)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\UCUNI\\AppData\\Local\\Temp/ipykernel_39608/3742051961.py:3: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
+      "  npact = np.array(actions)\n"
+     ]
+    }
+   ],
+   "source": [
+    "states = np.reshape(states, (24957, 90))\n",
+    "\n",
+    "npact = np.array(actions)\n",
+    "\n",
+    "last = npact[:,3]\n",
+    "newlast = []\n",
+    "last[2][0]\n",
+    "for i in range(len(last)):\n",
+    "    newlast.append(last[i][0])\n",
+    "\n",
+    "#print(newlast)\n",
+    "npact[:,3] = newlast\n",
+    "\n",
+    "statesNP = np.asarray(states)\n",
+    "actorProbsNP = np.asarray(actorProbs)\n",
+    "actionsNP = np.asarray(npact)\n",
+    "rewardsNP = np.asarray(rewards)\n",
+    "donesNP = np.asarray(dones)\n",
+    "thisSaveDir = \"GAIL-Expert-Data/1014-1302/pack-24957-RE.npz\"\n",
+    "\n",
+    "np.savez(\n",
+    "    thisSaveDir,\n",
+    "    states=statesNP,\n",
+    "    actorProbs=actorProbsNP,\n",
+    "    actions=actionsNP,\n",
+    "    rewards=rewardsNP,\n",
+    "    dones=donesNP,\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(actions)\n",
+    "npact = np.array(actions)\n",
+    "\n",
+    "last = npact[:,3]\n",
+    "newlast = []\n",
+    "last[2][0]\n",
+    "for i in range(len(last)):\n",
+    "    newlast.append(last[i][0])\n",
+    "\n",
+    "#print(newlast)\n",
+    "npact[:,3] = newlast\n",
+    "print(npact)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.7 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "86e2db13b09bd6be22cb599ea60c1572b9ef36ebeaa27a4c8e961d6df315ac32"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}