Fix bug with BlackJack environment

Author: pockerman

examples/example_1/example_1.cpp

@@ -47,7 +47,7 @@ void test_frozen_lake(RESTRLEnvClient& server){
 	std::cout<<"Environment version: "<<env.version()<<std::endl;
 
 	// once the env is created we can get it's id
-	std::cout<<"Environment is: "<<env.idx()<<std::endl;
+	std::cout<<"Environment idx is: "<<env.idx()<<std::endl;
 
 	// the create flag should be true
 	std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
@@ -107,8 +107,6 @@ void test_taxi(RESTRLEnvClient& server){
 
     Taxi env(server);
 
-	// environment name can also be accessed via env.env_name()
-	std::cout<<"Is environment registered: "<<server.is_env_registered(env.env_name())<<std::endl;
 	std::cout<<"Environment URL: "<<env.get_url()<<std::endl;
 
     // make the environment
@@ -121,7 +119,7 @@ void test_taxi(RESTRLEnvClient& server){
 	std::cout<<"Environment version: "<<env.version()<<std::endl;
 
 	// once the env is created we can get it's id
-	std::cout<<"Environment is: "<<env.idx()<<std::endl;
+	std::cout<<"Environment idx is: "<<env.idx()<<std::endl;
 
 	// the create flag should be true
 	std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
@@ -165,7 +163,6 @@ void test_taxi(RESTRLEnvClient& server){
 
     // close the environment
     env.close();
-
 }
 
 
@@ -207,12 +204,24 @@ void test_cliff_world(RESTRLEnvClient& server){
 	options["max_episode_steps"] = std::any(static_cast<bitrl::uint_t>(10));
 	std::unordered_map<std::string, std::any> reset_ops;
 	reset_ops.insert({"seed", static_cast<uint_t>(42)});
-    env.make("v0", options, reset_ops);
+    env.make("v1", options, reset_ops);
+
+	// query the environemnt version
+	std::cout<<"Environment version: "<<env.version()<<std::endl;
 
-    std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
-    std::cout<<"Is environment alive? "<<env.is_alive()<<std::endl;
-    std::cout<<"Number of valid actions? "<<env.n_actions()<<std::endl;
-    std::cout<<"Number of states? "<<env.n_states()<<std::endl;
+	// once the env is created we can get it's id
+	std::cout<<"Environment idx is: "<<env.idx()<<std::endl;
+
+	// the create flag should be true
+	std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
+
+	// environment should be alive on the server
+	std::cout<<"Is environment alive? "<<env.is_alive()<<std::endl;
+
+	// FrozenLake is a discrete state-action env so we can
+	// query number of actions and states
+	std::cout<<"Number of valid actions? "<<env.n_actions()<<std::endl;
+	std::cout<<"Number of states? "<<env.n_states()<<std::endl;
 
     // reset the environment
     auto time_step = env.reset();
@@ -265,10 +274,10 @@ int main(){
     example_1::test_taxi(server);
     std::cout<<"===================="<<std::endl;
     std::cout<<"Testing BlackJack..."<<std::endl;
-    //example_1::test_black_jack(server);
+    example_1::test_black_jack(server);
     std::cout<<"===================="<<std::endl;
     std::cout<<"Testing CliffWorld..."<<std::endl;
-    //example_1::test_cliff_world(server);
+    example_1::test_cliff_world(server);
     std::cout<<"===================="<<std::endl;
     return 0;
 }

src/bitrl/envs/gymnasium/toy_text/black_jack_env.cpp

@@ -39,14 +39,14 @@ namespace envs::gymnasium
 
 	BlackJack::BlackJack(network::RESTRLEnvClient& api_server)
 		:
-		ToyTextEnvBase<TimeStep<uint_t>, 48, 2>(api_server, BlackJack::name)
+		ToyTextEnvBase<TimeStep<std::vector<uint_t>>, 48, 2>(api_server, BlackJack::name)
 	{
 		this -> get_api_server().register_if_not(BlackJack::name, BlackJack::URI);
 	}
 
 	BlackJack::BlackJack(const BlackJack& other)
 		:
-		ToyTextEnvBase<TimeStep<uint_t>, 48, 2>(other),
+		ToyTextEnvBase<TimeStep<std::vector<uint_t>>, 48, 2>(other),
 		is_natural_(other.is_natural_),
 		is_sab_(other.is_sab_)
 	{}

src/bitrl/envs/gymnasium/toy_text/black_jack_env.h

@@ -23,11 +23,51 @@ namespace bitrl{
 namespace envs::gymnasium
 {
 
-	///
-/// \brief The BlackJack class. Wrapper to the Blackjack
-/// OpenAI-Gym environment.
-///
-class BlackJack final: public ToyTextEnvBase<TimeStep<uint_t>, 48, 2>
+
+/**
+* BlackJack class. Wrapper to the Blackjack OpenAI-Gym environment.
+*
+* This environment is part of the Toy Text environments which contains general information about the environment.
+* Action Space: Discrete(2)
+* Observation Space: Tuple(Discrete(32), Discrete(11), Discrete(2))
+* Blackjack is a card game where the goal is to beat the dealer by obtaining cards that sum to closer
+* to 21 (without going over 21) than the dealers cards.
+* The game starts with the dealer having one face up and one face down card, while the player has two face up cards.
+* All cards are drawn from an infinite deck (i.e. with replacement).
+* The card values are:
+* - Face cards (Jack, Queen, King) have a point value of 10.
+* - Aces can either count as 11 (called a ‘usable ace’) or 1.
+* - Numerical cards (2-10) have a value equal to their number.
+* The player has the sum of cards held.
+* The player can request additional cards (hit) until they decide to stop (stick) or exceed 21 (bust, immediate loss).
+* After the player sticks, the dealer reveals their facedown card, and draws cards until their sum is 17 or greater.
+* If the dealer goes bust, the player wins.
+* If neither the player nor the dealer busts, the outcome (win, lose, draw) is decided by whose sum is closer to 21.
+* This environment corresponds to the version of the blackjack problem described
+* in Example 5.1 in Reinforcement Learning: An Introduction by Sutton and Barto [1].
+*
+* Action Space
+* The action shape is (1,) in the range {0, 1} indicating whether to stick or hit.
+* 0: Stick
+* 1: Hit
+* Observation Space
+* The observation consists of a 3-tuple containing: the player’s current sum,
+* the value of the dealer’s one showing card (1-10 where 1 is ace), and whether the player holds a usable ace (0 or 1).
+* The observation is returned as (int(), int(), int()).
+* Rewards
+* win game: +1
+* lose game: -1
+* draw game: 0
+* win game with natural blackjack: +1.5 (if natural is True) +1 (if natural is False)
+* Episode End
+* The episode ends if the following happens:
+* Termination:
+* The player hits and the sum of hand exceeds 21.
+* The player sticks.
+* An ace will always be counted as usable (11) unless it busts the player.
+*
+*/
+class BlackJack final: public ToyTextEnvBase<TimeStep<std::vector<uint_t>>, 48, 2>
 {
 
 public:
@@ -45,7 +85,7 @@ class BlackJack final: public ToyTextEnvBase<TimeStep<uint_t>, 48, 2>
 		///
 	/// \brief The base type
 	///
-		typedef typename ToyTextEnvBase<TimeStep<uint_t>, 48, 2>::base_type base_type;
+		typedef typename ToyTextEnvBase<TimeStep<std::vector<uint_t>>, 48, 2>::base_type base_type;
 
 		///
 	/// \brief The time step type we return every time a step in the