# Continuous action space to control order quantity    self.action_space = spaces.Box(low=0, high=self.max_transport_capacity, shape=(1,), dtype=np.float32)    # Observation space: normalized inventory level, demand, and reorder point    self.observation_space = spaces.Box(low=0, high=1, shape=(3,), dtype=np.float32)    self.results_df = pd.DataFrame()  # DataFrame to store results    self.pending_orders = []  # Track pending orders (quantity and delivery day)    self.carry_over_backorder = 0  # Track backordered demand carried over    self.reset()def step(self, action):    """    Step through one day in the environment. Process the current day's demand, satisfy it using available    inventory, and carry over unmet demand to the next day if necessary.    """    row = self.data.iloc[self.current_day]    actual_demand = row['Demand']    # Add any carried-over backorder to today's demand    total_demand = actual_demand + self.carry_over_backorder    carrying_cost = row['Carrying Cost']    item_cost = row['Item Cost']    order_cost = row['Order Cost']    transport_emission_rate = row['Transport Emission Rate']    holding_emission_rate = row['Holding Emission Rate']    lead_time = row['Lead Time']    backorder_cost = row['Backorder Cost']    max_demand = self.data['Demand'].max()    avg_demand = self.data['Demand'].mean()    avg_lead_time = self.data['Lead Time'].mean()    holding_cost = item_cost * carrying_cost    safety_stock = (max_demand * avg_lead_time) - (avg_demand * avg_lead_time)    reorder_point = safety_stock + (avg_demand * lead_time)    # Initialize order_quantity to 0 in case no order is placed    order_quantity = 0    # Process pending orders    for order in self.pending_orders:        if self.current_day >= order['delivery_day']:            self.inventory_level += order['quantity']            print(f"Order delivered: {order['quantity']} units on day {self.current_day}")    self.pending_orders = [order for order in self.pending_orders if self.current_day < order['delivery_day']]    # Ensure inventory level does not exceed capacity    self.inventory_level = min(self.inventory_level, self.max_inventory_capacity)    # Calculate backorder before placing new orders    if total_demand > self.inventory_level:        backorder_quantity = total_demand - self.inventory_level        backorder_penalty = backorder_quantity * backorder_cost        self.inventory_level = 0        self.carry_over_backorder = backorder_quantity    else:        self.inventory_level -= total_demand        backorder_quantity = 0        backorder_penalty = 0        self.carry_over_backorder = 0    # Check if inventory is below the reorder point    if self.inventory_level <= reorder_point:        order_quantity = max(0, min(action[0], self.max_inventory_capacity - self.inventory_level))        if order_quantity > 0:            self.pending_orders.append({'quantity': order_quantity, 'delivery_day': self.current_day + lead_time})            print(f"Order placed: {order_quantity} units on day {self.current_day}")    # Calculate daily costs and emissions    holding_cost_total = max(0, self.inventory_level * holding_cost)    order_cost_total = max(0, order_quantity * item_cost + order_cost)    daily_cost = holding_cost_total + order_cost_total + backorder_penalty    holding_emissions = max(0, self.inventory_level * holding_emission_rate)    transport_emissions = max(0, (np.ceil(order_quantity / self.max_transport_capacity)) * transport_emission_rate)    daily_emissions = holding_emissions + transport_emissions    # Update totals    self.total_cost += daily_cost    self.total_emissions += daily_emissions    self.daily_costs.append(daily_cost)    self.daily_emissions.append(daily_emissions)    self.inventory_levels.append(self.inventory_level)    # Refined reward calculation    backorder_penalty_reward = -5.0 * backorder_quantity * backorder_cost    inventory_penalty = -0.5 * self.inventory_level    desired_inventory_level = reorder_point    efficiency_reward = -abs(self.inventory_level - desired_inventory_level) * 0.2    if self.inventory_level <= reorder_point:        order_incentive = 2.0    else:        order_incentive = -0.5    reward_cost = -daily_cost + backorder_penalty_reward + inventory_penalty + efficiency_reward + order_incentive    reward_emission = -daily_emissions    normalized_inventory = self.inventory_level / self.max_inventory_capacity    normalized_demand = total_demand / max_demand    normalized_reorder_point = reorder_point / self.max_inventory_capacity    normalized_observation = np.array([normalized_inventory, normalized_demand, normalized_reorder_point], dtype=np.float32)    daily_data = {        'Day': self.current_day + 1,        'Actual Demand': actual_demand,        'Carried Over Demand': self.carry_over_backorder,        'Inventory Level': self.inventory_level,        'Order Quantity': order_quantity if self.inventory_level <= reorder_point else 0,        'Holding Cost': holding_cost_total,        'Order Cost': order_cost_total,        'Backorder Penalty': backorder_penalty,        'Total Daily Cost': daily_cost,        'Daily Holding Emissions': holding_emissions,        'Daily Transport Emissions': transport_emissions,        'Total Daily Emissions': daily_emissions    }    self.results_df = pd.concat([self.results_df, pd.DataFrame([daily_data])], ignore_index=True)    self.current_day += 1    done = self.current_day >= len(self.data)    self.rewards.append((reward_cost, reward_emission))    return normalized_observation, (reward_cost, reward_emission), done, {}def reset(self):    self.current_day = 0    self.inventory_level = self.initial_inventory    self.total_cost = 0    self.total_emissions = 0    self.daily_costs = []    self.daily_emissions = []    self.inventory_levels = []    self.rewards = []    self.pending_orders = []    self.carry_over_backorder = 0    # Calculate initial values for safety_stock, avg_demand, and lead_time    max_demand = self.data['Demand'].max()    avg_demand = self.data['Demand'].mean()    avg_lead_time = self.data['Lead Time'].mean()    initial_lead_time = self.data.iloc[self.current_day]['Lead Time']    holding_cost = self.data.iloc[self.current_day]['Item Cost'] * self.data.iloc[self.current_day]['Carrying Cost']    safety_stock = (max_demand * avg_lead_time) - (avg_demand * avg_lead_time)    reorder_point = safety_stock + (avg_demand * initial_lead_time)  # Calculate reorder_point    # Initial observation    initial_demand = self.data.iloc[self.current_day]['Demand']    normalized_inventory = self.inventory_level / self.max_inventory_capacity    normalized_demand = initial_demand / max_demand    normalized_reorder_point = reorder_point / self.max_inventory_capacity  # Normalize reorder_point    return np.array([normalized_inventory, normalized_demand, normalized_reorder_point], dtype=np.float32)def render(self):    print(f"Final Total Cost: {self.total_cost}")    print(f"Final Total Emissions: {self.total_emissions}")    days = range(1, len(self.inventory_levels) + 1)    plt.figure(figsize=(20, 10))    plt.subplot(5, 1, 1)    plt.plot(days, self.inventory_levels, marker='o')    plt.title('Inventory Level Over Time')    plt.xlabel('Day')    plt.ylabel('Inventory Level')    plt.subplot(5, 1, 2)    plt.plot(days, self.daily_costs, marker='o')    plt.title('Daily Costs Over Time')    plt.xlabel('Day')    plt.ylabel('Costs')    plt.subplot(5, 1, 3)    plt.plot(days, self.daily_emissions, marker='o')    plt.title('Daily Emissions Over Time')    plt.xlabel('Day')    plt.ylabel('Emissions')    plt.subplot(5, 1, 4)    cost_rewards = [r[0] for r in self.rewards]    plt.plot(days, cost_rewards, marker='o', label='Cost Rewards')    plt.title('Cost Rewards Over Time')    plt.xlabel('Day')    plt.ylabel('Rewards')    plt.subplot(5, 1, 5)    emission_rewards = [r[1] for r in self.rewards]    plt.plot(days, emission_rewards, marker='o', label='Emission Rewards')    plt.title('Emission Rewards Over Time')    plt.xlabel('Day')    plt.ylabel('Rewards')    plt.tight_layout()    plt.show()def save_to_excel(self, file_name='inventory_output.xlsx'):    """Save the results DataFrame to an Excel file."""    if not self.results_df.empty:        self.results_df.to_excel(file_name, index=False)        print(f"Results saved to {file_name}")    else:        print("No results to save.")