package game.training;

import game.logic.GameBoard;
import game.logic.Vector;

public class SarsaLearning {

	public static double Alpha = 0.001; /* Learning rate */
	public static double Gamma = 0.8; 	/* Discount rate */

	/**
	 * The main method for training an agent. The method creates two new TrainPlayer
	 * instances (according to the type that was passed as a parameter), reads their
	 * weight vectors (for AGENT's or TRAINER's), and starts simulating games in a loop
	 * (number of games to simulate is given as an argument).
	 * The weights are saved in a log file (separately for each player) according to the
	 * sampling rate, which is given as a parameter. In any case, the last set of weights
	 * (end of session weights) are saved to the log file.
	 * At the end of the session, the method prints the scores for both players.
	 * 
	 * For debugging purposes, each time the weights are sampled to the log file,
	 * we print the game board and both player's features.
	 * In addition, we can watch a simulation of a whole game, by passing a game number
	 * as an argument.
	 * 
	 * @param rounds How many rounds of training should be simulated.
	 * @param statisticsPoint Our sampling rate, determines how often we write the weights to
	 * the log file. The final results will always be logged in the log file.
	 * @param watchGame The index of the game we want to watch (for debugging purposes).
	 * @param player1Type Defines the type of player1 (RANDOM, AGENT etc.)
	 * according to the types specified in the TrainPlayer class.
	 * @param player2Type Defines the type of player2 (RANDOM, AGENT etc.)
	 * according to the types specified in the TrainPlayer class.
	 * @param player1Depth The MiniMax tree depth for player1 (if necessary).
	 * @param player2Depth The MiniMax tree depth for player2 (if necessary).
	 * @param filename1 The file which contains player1's weights.
	 * @param filename2 The file which contains player2's weights.
	 * @param turns The number of turns we want to start our simulation with. This argument
	 * is crucial for updating Alpha, since we want to start later sessions with smaller Alpha
	 * value (use what we've learned).
	 */
	public static void TrainAgent(int rounds, int statisticsPoint,
			int watchGame, int player1Type, int player2Type, int player1Depth,
			int player2Depth, String filename1, String filename2, int turns) {
		GameSimulation g;
		int winner;
		int[] scores = new int[2];

		Vector weights1 = null;
		Vector weights2 = null;
		TrainPlayer player1 = null;
		TrainPlayer player2 = null;
		String player1Log = "Player1WeightsLog.txt";
		String player2Log = "Player2WeightsLog.txt";
		
		GameSimulation.TurnsPassed = turns;

		/* Start the players off with their weights - read weights from file */
		if (player1Type == TrainPlayer.AGENT
				|| player1Type == TrainPlayer.TRAINER) {
			weights1 = Vector.readfromFile(filename1);
		}

		if (player2Type == TrainPlayer.AGENT
				|| player2Type == TrainPlayer.TRAINER) {
			weights2 = Vector.readfromFile(filename2);
		}

		/* Create players to pass to the GameSimulation */
		if (player1Type == TrainPlayer.AGENT
				|| player1Type == TrainPlayer.TRAINER) {
			player1 = new TrainPlayer(GameBoard.RedPlayer, player1Type,
					player1Depth, weights1);
		} else {
			player1 = new TrainPlayer(GameBoard.RedPlayer);
		}

		if (player2Type == TrainPlayer.AGENT
				|| player2Type == TrainPlayer.TRAINER) {
			player2 = new TrainPlayer(GameBoard.BlackPlayer,
					player2Type, player2Depth, weights2);
		} else {
			player2 = new TrainPlayer(GameBoard.BlackPlayer);
		}

		/*
		 * The main training loop - Simulate games until rounds are over.
		 * Update the players' scores accordingly.
		 */
		for (int i = 0; i < rounds; i++) {
			
			/* Allows us to watch a full game of this training set - for debugging purposes */
			if (i == watchGame - 1) {
				g = new GameSimulation(player1, player2, true);
			} else {
				g = new GameSimulation(player1, player2);
			}

			winner = g.playGame();

			if (winner == 1) {
				scores[0]++;
			} else if (winner == 2) {
				scores[1]++;
			}

			if (statisticsPoint > 0 && i % statisticsPoint == 0) {
				System.out.println("Game #" + (i + 1) + ":");
				System.out.println("***********");
				GameBoard.printBoard(g.getGameBoard().getBoard());
				System.out.println("Game Vector:    "
						+ g.getGameBoard().getStateVector().indexVector());
				System.out.println("Player1 ForcedWins: " + g.getGameBoard().Player1ForcedWin);
				System.out.println("Player2 ForcedWins: " + g.getGameBoard().Player2ForcedWin);
				System.out.println("Player1 ProhibitedMoves: " + g.getGameBoard().Player1ProhibitedMoves);
				System.out.println("Player2 ProhibitedMoves: " + g.getGameBoard().Player2ProhibitedMoves);

				if (player1Type == TrainPlayer.AGENT) {
					Vector.writeToFile(player1Log, player1.getWeights());
				}

				if (player2Type == TrainPlayer.AGENT) {
					Vector.writeToFile(player2Log, player2.getWeights());
				}
			}
		}

		/* Write weights for agent players */
		if (player1Type == TrainPlayer.AGENT) {
			Vector.writeToFile(player1Log, player1.getWeights());
		}

		if (player2Type == TrainPlayer.AGENT) {
			Vector.writeToFile(player2Log, player2.getWeights());
		}

		/* Print final scores */
		System.out.println("RED score is: " + scores[0]);
		System.out.println("BLACK score is: " + scores[1]);

	}

	/**
	 * This method is the heart of the SARSA learning algorithm, and is responsible
	 * for updating the weights vectors. Given the proper parameters, the weight vector
	 * for the player can be updated according to the formula presented in the specification document.
	 * 
	 * @param weights A vector containing the weights that need to be updated.
	 * @param immediateReward The immediate reward received by performing action a in St.
	 * @param prevState The features vector holding the previous state, St.
	 * @param oldExpectedScore The expected return, Qt(st,at)
	 * @param currentExpectedScore The new expected return, Qt(at+1,st+1)
	 * @param watch A boolean that indicates whether we want to print data for debugging
	 * purposes.
	 * @param player the player that its weights we want to update.
	 * 
	 * @return A vector holding the updated weights for the player.
	 */
	public static Vector UpdateWeightVector(Vector weights,
			double immediateReward, Vector prevState, double oldExpectedScore,
			double currentExpectedScore, boolean watch, int player) {
		Vector updatedWeights = null;

		double error = immediateReward + Gamma * currentExpectedScore - oldExpectedScore;

		updatedWeights = Vector.MultiplyByScalar(prevState, Alpha * error);

		/* Debugging */
		if (watch) {
			System.out.println("Updating weights for player" + player);
			System.out.println("******************************");
			System.out.println("Previous state (St): "
					+ prevState.indexVector());
			System.out.println("Old Expected Score (Qt(St,at)): "
					+ oldExpectedScore);
			System.out.println("New Expected Score (Qt(St+1,at+1)): "
					+ currentExpectedScore);
			System.out.println("Immediate Reward: " + immediateReward);
			System.out.println("At (error) is: " + error);
			System.out.println("At * previousState = "
					+ updatedWeights.indexVector());

		}

		updatedWeights = Vector.AddVectors(updatedWeights, weights);
		
		/* Debugging */
		if (watch) {
			System.out.println("Old weights: " + weights.indexVector());
			System.out.println("New weights: " + updatedWeights.indexVector());
		}

		return updatedWeights;
	}
}
