问Y- aprob的卡氏Pong交叉熵/对数损失解释

EN

# forward the policy network and sample an action from the returned probability
  #########action 2 is up and 3 is down
  aprob, h = policy_forward(x)
  print("aprob\n {}\n h\n {}\n".format(aprob, h))
  #2 is up, 3 is down
  action = 2 if np.random.uniform() < aprob else 3 # roll the dice!
  print("action\n {}\n".format(action))
  # record various intermediates (needed later for backprop)
  xs.append(x) # observation, ie. the difference frame?
  #print("xs {}".format(xs))
  hs.append(h) # hidden state obtained from forward pass
  #print("hs {}".format(hs)) 
  #if action is up, y = 1, else 0
  y = 1 if action == 2 else 0 # a "fake label"
  print("y \n{}\n".format(y))
  dlogps.append(y - aprob) # grad that encourages the action that was taken to be taken (see http://cs231n.github.io/neural-networks-2/#losses if confused)
  print("dlogps\n {}\n".format(dlogps))
  # step the environment and get new measurements
  observation, reward, done, info = env.step(action)
  print("observation\n {}\n reward\n {}\n done\n {}\n ".format(observation, reward, done))
  reward_sum += reward
  print("reward_sum\n {}\n".format(reward_sum))
  drs.append(reward) # record reward (has to be done after we call step() to get reward for previous action)
  print("drs\n {}\n".format(drs))
  if done: # an episode finished
    episode_number += 1

#forward pass, how is logp a logp without any log operation?????
def policy_forward(x):
  h = np.dot(model['W1'], x)
  h[h<0] = 0 # ReLU nonlinearity
  logp = np.dot(model['W2'], h)
  p = sigmoid(logp)
  #print("p\n {}\n and h\n {}\n".format(p, h))
  return p, h # return probability of taking action 2 (up), and hidden state

aprob
 0.5

action
 3

aprob
 0.5010495775824385

action
 2

aprob
 0.5023498477623756

action
 2

aprob
 0.5051575154468827

action
 2