DI-engine/dizoo/beergame/envs/clBeergame.py

# Code Reference: https://github.com/OptMLGroup/DeepBeerInventory-RL.
import numpy as np
from random import randint
from .BGAgent import Agent
from matplotlib import rc
rc('text', usetex=True)
from .plotting import plotting, savePlot
import matplotlib.pyplot as plt
import os
import time
from time import gmtime, strftime


class clBeerGame(object):

    def __init__(self, config):
        self.config = config
        self.curGame = 0  # The number associated with the current game (counter of the game)
        self.curTime = 0
        self.totIterPlayed = 0  # total iterations of the game, played so far in this and previous games
        self.players = self.createAgent()  # create the agents
        self.T = 0
        self.demand = []
        self.ifOptimalSolExist = self.config.ifOptimalSolExist
        self.getOptimalSol()
        self.totRew = 0  # it is reward of all players obtained for the current player.
        self.resultTest = []
        self.runnerMidlResults = []  # stores the results to use in runner comparisons
        self.runnerFinlResults = []  # stores the results to use in runner comparisons
        self.middleTestResult = [
        ]  # stores the whole middle results of bs, Strm, and random to avoid doing same tests multiple of times.
        self.runNumber = 0  # the runNumber which is used when use runner
        self.strNum = 0  # the runNumber which is used when use runner

    # createAgent : Create agent objects (agentNum,IL,OO,c_h,c_p,type,config)
    def createAgent(self):
        agentTypes = self.config.agentTypes
        return [
            Agent(
                i, self.config.ILInit[i], self.config.AOInit, self.config.ASInit[i], self.config.c_h[i],
                self.config.c_p[i], self.config.eta[i], agentTypes[i], self.config
            ) for i in range(self.config.NoAgent)
        ]

    # planHorizon : Find a random planning horizon
    def planHorizon(self):
        # TLow: minimum number for the planning horizon # TUp: maximum number for the planning horizon
        # output: The planning horizon which is chosen randomly.
        return randint(self.config.TLow, self.config.TUp)

    # this function resets the game for start of the new game
    def resetGame(self, demand: np.ndarray):
        self.demand = demand
        self.curTime = 0
        self.curGame += 1
        self.totIterPlayed += self.T
        self.T = self.planHorizon()
        # reset the required information of player for each episode
        for k in range(0, self.config.NoAgent):
            self.players[k].resetPlayer(self.T)

        # update OO when there are initial IL,AO,AS
        self.update_OO()

    # correction on cost at time T according to the cost of the other players
    def getTotRew(self):
        totRew = 0
        for i in range(self.config.NoAgent):
            # sum all rewards for the agents and make correction
            totRew += self.players[i].cumReward

        for i in range(self.config.NoAgent):
            self.players[i].curReward += self.players[i].eta * (totRew - self.players[i].cumReward)  # /(self.T)

    # make correction to the rewards in the experience replay for all iterations of current game
    def distTotReward(self, role: int):
        totRew = 0
        optRew = 0.1  # why?
        for i in range(self.config.NoAgent):
            # sum all rewards for the agents and make correction
            totRew += self.players[i].cumReward
        totRew += optRew

        return totRew, self.players[role].cumReward

    def getAction(self, k: int, action: np.ndarray, playType="train"):
        if playType == "train":
            if self.players[k].compType == "srdqn":
                self.players[k].action = np.zeros(self.config.actionListLen)
                self.players[k].action[action] = 1
            elif self.players[k].compType == "Strm":
                self.players[k].action = np.zeros(self.config.actionListLenOpt)
                self.players[k].action[np.argmin(np.abs(np.array(self.config.actionListOpt)\
                    - max(0, round(self.players[k].AO[self.curTime] + \
                    self.players[k].alpha_b*(self.players[k].IL - self.players[k].a_b) + \
                    self.players[k].betta_b*(self.players[k].OO - self.players[k].b_b)))))] = 1
            elif self.players[k].compType == "rnd":
                self.players[k].action = np.zeros(self.config.actionListLen)
                a = np.random.randint(self.config.actionListLen)
                self.players[k].action[a] = 1
            elif self.players[k].compType == "bs":
                self.players[k].action = np.zeros(self.config.actionListLenOpt)
                if self.config.demandDistribution == 2:
                    if self.curTime and self.config.use_initial_BS <= 4:
                        self.players[k].action[np.argmin(np.abs(np.array(self.config.actionListOpt) - \
                        max(0, (self.players[k].int_bslBaseStock - (self.players[k].IL + self.players[k].OO - self.players[k].AO[self.curTime])))))] = 1
                    else:
                        self.players[k].action[np.argmin(np.abs(np.array(self.config.actionListOpt) - \
                        max(0, (self.players[k].bsBaseStock - (self.players[k].IL + self.players[k].OO - self.players[k].AO[self.curTime])))))] = 1
                else:
                    self.players[k].action[np.argmin(np.abs(np.array(self.config.actionListOpt) - \
                    max(0, (self.players[k].bsBaseStock - (self.players[k].IL + self.players[k].OO - self.players[k].AO[self.curTime])))))] = 1
        elif playType == "test":
            if self.players[k].compTypeTest == "srdqn":
                self.players[k].action = np.zeros(self.config.actionListLen)
                self.players[k].action = self.players[k].brain.getDNNAction(self.playType)
            elif self.players[k].compTypeTest == "Strm":
                self.players[k].action = np.zeros(self.config.actionListLenOpt)

                self.players[k].action[np.argmin(np.abs(np.array(self.config.actionListOpt)-\
                    max(0,round(self.players[k].AO[self.curTime] +\
                     self.players[k].alpha_b*(self.players[k].IL - self.players[k].a_b) +\
                     self.players[k].betta_b*(self.players[k].OO - self.players[k].b_b)))))] = 1
            elif self.players[k].compTypeTest == "rnd":
                self.players[k].action = np.zeros(self.config.actionListLen)
                a = np.random.randint(self.config.actionListLen)
                self.players[k].action[a] = 1
            elif self.players[k].compTypeTest == "bs":
                self.players[k].action = np.zeros(self.config.actionListLenOpt)

                if self.config.demandDistribution == 2:
                    if self.curTime and self.config.use_initial_BS <= 4:
                        self.players[k].action [np.argmin(np.abs(np.array(self.config.actionListOpt)-\
                          max(0,(self.players[k].int_bslBaseStock - (self.players[k].IL + self.players[k].OO - self.players[k].AO[self.curTime]))) ))] = 1
                    else:
                        self.players[k].action [np.argmin(np.abs(np.array(self.config.actionListOpt)-\
                          max(0,(self.players[k].bsBaseStock - (self.players[k].IL + self.players[k].OO - self.players[k].AO[self.curTime]))) ))] = 1
                else:
                    self.players[k].action [np.argmin(np.abs(np.array(self.config.actionListOpt)-\
                       max(0,(self.players[k].bsBaseStock - (self.players[k].IL + self.players[k].OO - self.players[k].AO[self.curTime]))) ))] = 1
        else:
            # not a valid player is defined.
            raise Exception('The player type is not defined or it is not a valid type.!')

    def next(self):
        # get a random leadtime
        leadTimeIn = randint(
            self.config.leadRecItemLow[self.config.NoAgent - 1], self.config.leadRecItemUp[self.config.NoAgent - 1]
        )
        # handle the most upstream recieved shipment
        self.players[self.config.NoAgent - 1].AS[self.curTime +
                                                 leadTimeIn] += self.players[self.config.NoAgent -
                                                                             1].actionValue(self.curTime)

        for k in range(self.config.NoAgent - 1, -1, -1):  # [3,2,1,0]

            # get current IL and Backorder
            current_IL = max(0, self.players[k].IL)
            current_backorder = max(0, -self.players[k].IL)

            # TODO: We have get the AS and AO from the UI and update our AS and AO, so that code update the corresponding variables

            # increase IL and decrease OO based on the action, for the next period
            self.players[k].recieveItems(self.curTime)

            # observe the reward
            possible_shipment = min(
                current_IL + self.players[k].AS[self.curTime], current_backorder + self.players[k].AO[self.curTime]
            )

            # plan arrivals of the items to the downstream agent
            if self.players[k].agentNum > 0:
                leadTimeIn = randint(self.config.leadRecItemLow[k - 1], self.config.leadRecItemUp[k - 1])
                self.players[k - 1].AS[self.curTime + leadTimeIn] += possible_shipment

            # update IL
            self.players[k].IL -= self.players[k].AO[self.curTime]
            # observe the reward
            self.players[k].getReward()
            self.players[k].hist[-1][-2] = self.players[k].curReward
            self.players[k].hist2[-1][-2] = self.players[k].curReward

            # update next observation
            self.players[k].nextObservation = self.players[k].getCurState(self.curTime + 1)

        if self.config.ifUseTotalReward:
            # correction on cost at time T
            if self.curTime == self.T:
                self.getTotRew()

        self.curTime += 1

    def handelAction(self, action: np.ndarray, playType="train"):
        # get random lead time
        leadTime = randint(self.config.leadRecOrderLow[0], self.config.leadRecOrderUp[0])
        # set AO
        self.players[0].AO[self.curTime] += self.demand[self.curTime]
        for k in range(0, self.config.NoAgent):
            self.getAction(k, action, playType)

            self.players[k].srdqnBaseStock += [self.players[k].actionValue( \
             self.curTime) + self.players[k].IL + self.players[k].OO]

            # update hist for the plots
            self.players[k].hist += [[self.curTime, self.players[k].IL, self.players[k].OO,\
               self.players[k].actionValue(self.curTime), self.players[k].curReward, self.players[k].srdqnBaseStock[-1]]]

            if self.players[k].compType == "srdqn":
                self.players[k].hist2 += [[self.curTime, self.players[k].IL, self.players[k].OO, self.players[k].AO[self.curTime], self.players[k].AS[self.curTime], \
                  self.players[k].actionValue(self.curTime), self.players[k].curReward, \
                  self.config.actionList[np.argmax(self.players[k].action)]]]

            else:
                self.players[k].hist2 += [[self.curTime, self.players[k].IL, self.players[k].OO, self.players[k].AO[self.curTime], self.players[k].AS[self.curTime], \
                  self.players[k].actionValue(self.curTime), self.players[k].curReward, 0]]

            # updates OO and AO at time t+1
            self.players[k].OO += self.players[k].actionValue(self.curTime)  # open order level update
            leadTime = randint(self.config.leadRecOrderLow[k], self.config.leadRecOrderUp[k])
            if self.players[k].agentNum < self.config.NoAgent - 1:
                self.players[k + 1].AO[self.curTime + leadTime] += self.players[k].actionValue(
                    self.curTime
                )  # open order level update

    # check the Shang and Song (2003) condition, and if it works, obtains the base stock policy values for each agent
    def getOptimalSol(self):
        # if self.config.NoAgent !=1:
        if self.config.NoAgent != 1 and 1 == 2:
            # check the Shang and Song (2003) condition.
            for k in range(self.config.NoAgent - 1):
                if not (self.players[k].c_h == self.players[k + 1].c_h and self.players[k + 1].c_p == 0):
                    self.ifOptimalSolExist = False

            # if the Shang and Song (2003) condition satisfied, it runs the algorithm
            if self.ifOptimalSolExist == True:
                calculations = np.zeros((7, self.config.NoAgent))
                for k in range(self.config.NoAgent):
                    # DL_high
                    calculations[0][k] = ((self.config.leadRecItemLow + self.config.leadRecItemUp + 2) / 2 \
                           + (self.config.leadRecOrderLow + self.config.leadRecOrderUp + 2) / 2) * \
                         (self.config.demandUp - self.config.demandLow - 1)
                    if k > 0:
                        calculations[0][k] += calculations[0][k - 1]
                    # probability_high
                    nominator_ch = 0
                    low_denominator_ch = 0
                    for j in range(k, self.config.NoAgent):
                        if j < self.config.NoAgent - 1:
                            nominator_ch += self.players[j + 1].c_h
                        low_denominator_ch += self.players[j].c_h
                    if k == 0:
                        high_denominator_ch = low_denominator_ch
                    calculations[2][k] = (self.players[0].c_p +
                                          nominator_ch) / (self.players[0].c_p + low_denominator_ch + 0.0)
                    # probability_low
                    calculations[3][k] = (self.players[0].c_p +
                                          nominator_ch) / (self.players[0].c_p + high_denominator_ch + 0.0)
                # S_high
                calculations[4] = np.round(np.multiply(calculations[0], calculations[2]))
                # S_low
                calculations[5] = np.round(np.multiply(calculations[0], calculations[3]))
                # S_avg
                calculations[6] = np.round(np.mean(calculations[4:6], axis=0))
                # S', set the base stock values into each agent.
                for k in range(self.config.NoAgent):
                    if k == 0:
                        self.players[k].bsBaseStock = calculations[6][k]

                    else:
                        self.players[k].bsBaseStock = calculations[6][k] - calculations[6][k - 1]
                        if self.players[k].bsBaseStock < 0:
                            self.players[k].bsBaseStock = 0
        elif self.config.NoAgent == 1:
            if self.config.demandDistribution == 0:
                self.players[0].bsBaseStock = np.ceil(
                    self.config.c_h[0] / (self.config.c_h[0] + self.config.c_p[0] + 0.0)
                ) * ((self.config.demandUp - self.config.demandLow - 1) / 2) * self.config.leadRecItemUp
        elif 1 == 1:
            f = self.config.f
            f_init = self.config.f_init
            for k in range(self.config.NoAgent):
                self.players[k].bsBaseStock = f[k]
                self.players[k].int_bslBaseStock = f_init[k]

    def update_OO(self):
        for k in range(0, self.config.NoAgent):
            if k < self.config.NoAgent - 1:
                self.players[k].OO = sum(self.players[k + 1].AO) + sum(self.players[k].AS)
            else:
                self.players[k].OO = sum(self.players[k].AS)

    def doTestMid(self, demandTs):
        self.resultTest = []
        m = strftime("%Y-%m-%d-%H-%M-%S", gmtime())
        self.doTest(m, demandTs)
        print("---------------------------------------------------------------------------------------")
        resultSummary = np.array(self.resultTest).mean(axis=0).tolist()

        result_srdqn = ', '.join(map("{:.2f}".format, resultSummary[0]))
        result_rand = ', '.join(map("{:.2f}".format, resultSummary[1]))
        result_strm = ', '.join(map("{:.2f}".format, resultSummary[2]))
        if self.ifOptimalSolExist:
            result_bs = ', '.join(map("{:.2f}".format, resultSummary[3]))
            print(
                'SUMMARY; {0:s}; ITER= {1:d}; OURPOLICY= [{2:s}]; SUM = {3:2.4f}; Rand= [{4:s}]; SUM = {5:2.4f}; STRM= [{6:s}]; SUM = {7:2.4f}; BS= [{8:s}]; SUM = {9:2.4f}'
                .format(
                    strftime("%Y-%m-%d %H:%M:%S", gmtime()), self.curGame, result_srdqn, sum(resultSummary[0]),
                    result_rand, sum(resultSummary[1]), result_strm, sum(resultSummary[2]), result_bs,
                    sum(resultSummary[3])
                )
            )

        else:
            print(
                'SUMMARY; {0:s}; ITER= {1:d}; OURPOLICY= [{2:s}]; SUM = {3:2.4f}; Rand= [{4:s}]; SUM = {5:2.4f}; STRM= [{6:s}]; SUM = {7:2.4f}'
                .format(
                    strftime("%Y-%m-%d %H:%M:%S", gmtime()), self.curGame, result_srdqn, sum(resultSummary[0]),
                    result_rand, sum(resultSummary[1]), result_strm, sum(resultSummary[2])
                )
            )

        print("=======================================================================================")

    def doTest(self, m, demand):
        import matplotlib.pyplot as plt
        if self.config.ifSaveFigure:
            plt.figure(self.curGame, figsize=(12, 8), dpi=80, facecolor='w', edgecolor='k')

            # self.demand = demand
            # use dnn to get output.
            Rsltdnn, plt = self.tester(self.config.agentTypes, plt, 'b', 'OurPolicy', m)
            baseStockdata = self.players[0].srdqnBaseStock
            # # use random to get output.
            RsltRnd, plt = self.tester(["rnd", "rnd", "rnd", "rnd"], plt, 'y', 'RAND', m)

            # use formual to get output.
            RsltStrm, plt = self.tester(["Strm", "Strm", "Strm", "Strm"], plt, 'g', 'Strm', m)

            # use optimal strategy to get output, if it works.
            if self.ifOptimalSolExist:
                if self.config.agentTypes == ["srdqn", "Strm", "Strm", "Strm"]:
                    Rsltbs, plt = self.tester(["bs", "Strm", "Strm", "Strm"], plt, 'r', 'Strm-BS', m)
                elif self.config.agentTypes == ["Strm", "srdqn", "Strm", "Strm"]:
                    Rsltbs, plt = self.tester(["Strm", "bs", "Strm", "Strm"], plt, 'r', 'Strm-BS', m)
                elif self.config.agentTypes == ["Strm", "Strm", "srdqn", "Strm"]:
                    Rsltbs, plt = self.tester(["Strm", "Strm", "bs", "Strm"], plt, 'r', 'Strm-BS', m)
                elif self.config.agentTypes == ["Strm", "Strm", "Strm", "srdqn"]:
                    Rsltbs, plt = self.tester(["Strm", "Strm", "Strm", "bs"], plt, 'r', 'Strm-BS', m)
                elif self.config.agentTypes == ["srdqn", "rnd", "rnd", "rnd"]:
                    Rsltbs, plt = self.tester(["bs", "rnd", "rnd", "rnd"], plt, 'r', 'RND-BS', m)
                elif self.config.agentTypes == ["rnd", "srdqn", "rnd", "rnd"]:
                    Rsltbs, plt = self.tester(["rnd", "bs", "rnd", "rnd"], plt, 'r', 'RND-BS', m)
                elif self.config.agentTypes == ["rnd", "rnd", "srdqn", "rnd"]:
                    Rsltbs, plt = self.tester(["rnd", "rnd", "bs", "rnd"], plt, 'r', 'RND-BS', m)
                elif self.config.agentTypes == ["rnd", "rnd", "rnd", "srdqn"]:
                    Rsltbs, plt = self.tester(["rnd", "rnd", "rnd", "bs"], plt, 'r', 'RND-BS', m)
                else:
                    Rsltbs, plt = self.tester(["bs", "bs", "bs", "bs"], plt, 'r', 'BS', m)
            # hold the results of the optimal solution
                self.middleTestResult += [[RsltRnd, RsltStrm, Rsltbs]]
            else:
                self.middleTestResult += [[RsltRnd, RsltStrm]]

        else:
            # return the obtained results into their lists
            RsltRnd = self.middleTestResult[m][0]
            RsltStrm = self.middleTestResult[m][1]
            if self.ifOptimalSolExist:
                Rsltbs = self.middleTestResult[m][2]

        # save the figure
        if self.config.ifSaveFigure:
            savePlot(self.players, self.curGame, Rsltdnn, RsltStrm, Rsltbs, RsltRnd, self.config, m)
            plt.close()

        result_srdqn = ', '.join(map("{:.2f}".format, Rsltdnn))
        result_rand = ', '.join(map("{:.2f}".format, RsltRnd))
        result_strm = ', '.join(map("{:.2f}".format, RsltStrm))
        if self.ifOptimalSolExist:
            result_bs = ', '.join(map("{:.2f}".format, Rsltbs))
            print(
                'output; {0:s}; Iter= {1:s}; SRDQN= [{2:s}]; sum = {3:2.4f}; Rand= [{4:s}]; sum = {5:2.4f}; Strm= [{6:s}]; sum = {7:2.4f}; BS= [{8:s}]; sum = {9:2.4f}'
                .format(
                    strftime("%Y-%m-%d %H:%M:%S", gmtime()), str(str(self.curGame) + "-" + str(m)), result_srdqn,
                    sum(Rsltdnn), result_rand, sum(RsltRnd), result_strm, sum(RsltStrm), result_bs, sum(Rsltbs)
                )
            )
            self.resultTest += [[Rsltdnn, RsltRnd, RsltStrm, Rsltbs]]

        else:
            print(
                'output; {0:s}; Iter= {1:s}; SRDQN= [{2:s}]; sum = {3:2.4f}; Rand= [{4:s}]; sum = {5:2.4f}; Strm= [{6:s}]; sum = {7:2.4f}'
                .format(
                    strftime("%Y-%m-%d %H:%M:%S", gmtime()), str(str(self.curGame) + "-" + str(m)), result_srdqn,
                    sum(Rsltdnn), result_rand, sum(RsltRnd), result_strm, sum(RsltStrm)
                )
            )

            self.resultTest += [[Rsltdnn, RsltRnd, RsltStrm]]

        return sum(Rsltdnn)

    def tester(self, testType, plt, colori, labeli, m):

        # set computation type for test
        for k in range(0, self.config.NoAgent):
            # self.players[k].compTypeTest = testType[k]
            self.players[k].compType = testType[k]
        # run the episode to get the results.
        if labeli != 'OurPolicy':
            result = self.playGame(self.demand)
        else:
            result = [-1 * self.players[i].cumReward for i in range(0, self.config.NoAgent)]
        # add the results into the figure
        if self.config.ifSaveFigure:
            plt = plotting(plt, [np.array(self.players[i].hist) for i in range(0, self.config.NoAgent)], colori, labeli)
        if self.config.ifsaveHistInterval and ((self.curGame == 0) or (self.curGame == 1) or (self.curGame == 2) or (self.curGame == 3) or ((self.curGame - 1) % self.config.saveHistInterval == 0)\
         or ((self.curGame) % self.config.saveHistInterval == 0) or ((self.curGame) % self.config.saveHistInterval == 1) \
         or ((self.curGame) % self.config.saveHistInterval == 2)) :
            for k in range(0, self.config.NoAgent):
                name = labeli + "-" + str(self.curGame) + "-" + "player" + "-" + str(k) + "-" + str(m)
                np.save(os.path.join(self.config.model_dir, name), np.array(self.players[k].hist2))

        # save the figure of base stocks
        # if self.config.ifSaveFigure and (self.curGame in range(self.config.saveFigInt[0],self.config.saveFigInt[1])):
        # 	for k in range(self.config.NoAgent):
        # 		if self.players[k].compTypeTest == 'dnn':
        # 			plotBaseStock(self.players[k].srdqnBaseStock, 'b', 'base stock of agent '+ str(self.players[k].agentNum), self.curGame, self.config, m)

        return result, plt

    def playGame(self, demand):
        self.resetGame(demand)

        # run the game
        while self.curTime < self.T:
            self.handelAction(np.array(0))  # action won't be used.
            self.next()
        return [-1 * self.players[i].cumReward for i in range(0, self.config.NoAgent)]