package player.gamer.statemachine.sadna;


/**
 * Update vector weights with TD(0) algorithm
 */
public class TDWeightUpdater implements WeightUpdater {

	private int[] lastRoundAttrVec = new int[Board.ATTR_VEC_SIZE];
	private double[] weights;
	private double discountedFactor = 0.9; // Gama
	private double learningRate = 0.0001; // Alpha
	private double lastRoundScore;
	
	public TDWeightUpdater(double[] startingWeights) {
		this.weights = startingWeights;
	}
	
	public double[] getWeights() {
		return weights;
	}
	
	public void updateWeights(Board board, int reward) {
		int[] currentAttrVec = board.calclateAttributeVector();
		
		// calculate the error
		double score = 0; // Vt(St+1) = last round weights * current board attributes
		for (int i = 0; i < weights.length; i++) {
			score += (weights[i] * currentAttrVec[i]);
		}
		double error = reward + (discountedFactor*score) - lastRoundScore; // At = r+gama*Vt(St+1)-Vt(St)

		// update the new weights vector
		for (int i = 0; i < weights.length; i++) {
			weights[i] += (learningRate * error * lastRoundAttrVec[i]); // Wt+1 = Wt + alpha*At*Vec(St)
		}

		// calculate new score (for the next round)
		lastRoundScore = 0;
		for (int i = 0; i < weights.length; i++) {
			lastRoundScore += (weights[i] * currentAttrVec[i]); // Vt+1(St+1) = new weights * current board attributes
		}
		lastRoundAttrVec = currentAttrVec;
	}
}
