Var 与梯度

w = torch.tensor([1.0, 2.0], requires_grad=True)
k = torch.tensor([10.0, 20.0])
h = w * k
loss = h.sum()
loss.backward()
print("w.grad", w.grad, "k.grad", k.grad)
print("h.grad", h.grad)

let k = Tensor::new(&[10f32, 20.], &dev)?;
let w = Var::new(&[1f32, 2.], &dev)?;
let h = (w.as_tensor() * &k)?;
let loss = h.sum_all()?;
let g = loss.backward()?;
println!("w.grad {:?} k.grad {:?} h.grad {:?}", v(g.get(&w)), v(g.get(&k)), v(g.get(&h)));

w.grad tensor([10., 20.]) k.grad None
h.grad None

w.grad Some([10.0, 20.0]) k.grad Some([1.0, 2.0]) h.grad None

error[E0369]: cannot multiply `Var` by `{float}`
  --> src/bin/nonleaf.rs:5:52
   |
 5 |   let w = (Var::ones(2, DType::F32, &Device::Cpu)? * 0.5)?;
   |            --------------------------------------- ^ --- {float}
   |            |
   |            Var
   |
note: `Var` does not implement `std::ops::Mul<{float}>`
  --> .../candle-core-0.11.0/src/variable.rs:10:1
   |
10 | pub struct Var(Tensor);
   | ^^^^^^^^^^^^^^ `Var` is defined in another crate

fn forward(p: &[Var], x: &Tensor) -> Result<Tensor> {
  let h = x.matmul(&p[0].t()?)?.broadcast_add(&p[1])?.relu()?;
  Ok(h.matmul(&p[2].t()?)?.broadcast_add(&p[3])?)
}

let p = vars(&r, &dev)?; // 每个参数 Var::from_tensor
let loss = candle_nn::loss::mse(&forward(&p, &x)?, &y)?;
let grads = loss.backward()?;
let g: Vec<Tensor> = p.iter().map(|v| grads.get(v).cloned().context("grad")).collect::<Result<_>>()?;

loss diff 2.3841858e-7
  grad.fc1.weight: 0e0
  grad.fc1.bias: 2.9802322e-8
  grad.fc2.weight: 0e0
  grad.fc2.bias: 4.4703484e-8
grad worst 4.4703484e-8

loss diff 2.3841858e-7
  grad.fc1.weight: 2.9802322e-8
  grad.fc1.bias: 1.4901161e-8
  grad.fc2.weight: 2.9802322e-8
  grad.fc2.bias: 2.9802322e-8
grad worst 2.9802322e-8

try:
  loss.backward()          # 上面那个 loss 的第二次
except RuntimeError as e:
  print("RuntimeError:", str(e).splitlines()[0])

w = torch.tensor([1.0, 2.0], requires_grad=True)
loss = (w * k).sum()
loss.backward(retain_graph=True)
loss.backward()
print("w.grad after two backward", w.grad)

RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.
w.grad after two backward tensor([20., 40.])

let g2 = loss.backward()?;
println!("second w.grad {:?}", v(g2.get(&w)));

let mut acc = loss.backward()?;
acc.extend(loss.backward()?)?;
println!("w.grad after extend {:?}", v(acc.get(&w)));

second w.grad Some([10.0, 20.0])
w.grad after extend Some([20.0, 40.0])

w = torch.tensor([1.0, 2.0], requires_grad=True)
h = w * k
h.retain_grad()
(h * h).sum().backward()
print("h.grad", h.grad, "w.grad", w.grad)

h.grad tensor([20., 80.]) w.grad tensor([ 200., 1600.])

let z = Var::zeros(2, DType::F32, &dev)?;
let h = ((w.as_tensor() * &k)? + z.as_tensor())?;
let g = (&h * &h)?.sum_all()?.backward()?;
println!("h.grad {:?} z.grad {:?} w.grad {:?}", v(g.get(&h)), v(g.get(&z)), v(g.get(&w)));

h.grad None z.grad Some([20.0, 80.0]) w.grad Some([200.0, 1600.0])

w = torch.tensor([1.0, 2.0], requires_grad=True)
(w * k).backward()                          # 非标量

torch.tensor([1, 2], requires_grad=True)    # 整数

w = torch.ones(2, requires_grad=True) * 0.5 # 非叶子
(w * k).sum().backward()
print("w.is_leaf", w.is_leaf, "w.grad", w.grad)
torch.optim.SGD([w], lr=0.1)

RuntimeError: grad can be implicitly created only for scalar outputs
RuntimeError: Only Tensors of floating point and complex dtype can require gradients
w.is_leaf False w.grad None
ValueError: can't optimize a non-leaf Tensor

let g = (w.as_tensor() * &k)?.backward()?;                 // 非标量
println!("w.grad {:?}", v(g.get(&w)));

let wi = Var::new(&[1u32, 2], &dev)?;                      // 整数
println!("Var::new(u32) ok, dtype {:?}", wi.dtype());
let g = (wi.to_dtype(DType::F32)? * &k)?.sum_all()?.backward()?;
println!("wi.grad is_some {}", g.get(&wi).is_some());

w.grad Some([10.0, 20.0])
Var::new(u32) ok, dtype U32
wi.grad is_some false

let w = (Var::ones(2, DType::F32, &Device::Cpu)?.as_tensor() * 0.5)?;
let _opt = SGD::new(vec![w], 0.1)?;

error[E0308]: mismatched types
 --> src/bin/nonleaf.rs:6:28
  |
6 |   let _opt = SGD::new(vec![w], 0.1)?;
  |                            ^ expected `Var`, found `Tensor`

t = torch.tensor([1.0, 2.0])
p = t.requires_grad_()
print("same object", p is t)

same object True

let t = Tensor::new(&[1f32, 2.], &dev)?;
let p = Var::from_tensor(&t)?;
p.set(&Tensor::zeros(2, DType::F32, &dev)?)?;
println!("t {:?} p {:?} same id {}", t.to_vec1::<f32>()?, p.to_vec1::<f32>()?, t.id() == p.id());
let p2 = Var::from_tensor(p.as_tensor())?;
println!("from_tensor(var) same id {}", p2.id() == p.id());

t [1.0, 2.0] p [0.0, 0.0] same id false
from_tensor(var) same id true