Commit596e496

committed

move and include zeros_like op

1 parent0119de5 commit596e496Copy full SHA for 596e496

File tree

-3

lines changed

-3

lines changed

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -1038,7 +1038,7 @@ std::tuple<Tensor, Tensor, Tensor, Tensor> _scaled_dot_product_efficient_attenti`
`1038`	`1038`	`for (int i =1; i < log_sumexp.dim(); i++) {`
`1039`	`1039`	`lse_sizes.push_back(log_sumexp.size(i));`
`1040`	`1040`	`}`
`1041`		`- final_log_sumexp =at::empty(lse_sizes, log_sumexp.options());`
	`1041`	`+ final_log_sumexp =at::empty(std::move(lse_sizes), log_sumexp.options());`
`1042`	`1042`	`final_log_sumexp.slice(0, start, end).copy_(log_sumexp);`
`1043`	`1043`	`}`
`1044`	`1044`

Lines changed: 3 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@`
`24`	`24`	`#include<ATen/Functions.h>`
`25`	`25`	`#include<ATen/NativeFunctions.h>`
`26`	`26`	`#else`
	`27`	`+#include<ATen/ops/zeros_like.h>`
`27`	`28`	`#include<ATen/ops/empty_strided.h>`
`28`	`29`	`#include<ATen/ops/_flash_attention_backward.h>`
`29`	`30`	`#include<ATen/ops/_flash_attention_backward_native.h>`
`@@ -964,7 +965,7 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor, at::Tensor> _scaled_dot_product_e`
`964`	`965`	`scale,`
`965`	`966`	`std::nullopt);// num_split_keys`
`966`	`967`	`returnstd::make_tuple(`
`967`		`- grad_q.transpose(1,2), grad_k.transpose(1,2), grad_v.transpose(1,2), grad_bias);`
	`968`	`+ grad_q.transpose(1,2), grad_k.transpose(1,2), grad_v.transpose(1,2),std::move(grad_bias));`
`968`	`969`	`};`
`969`	`970`
`970`	`971`	`// process in chunks if batch size exceeds maximum`
`@@ -982,7 +983,7 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor, at::Tensor> _scaled_dot_product_e`
`982`	`983`	`for (int i =1; i < dim; i++) {`
`983`	`984`	`sizes.push_back(tensor.size(i));`
`984`	`985`	`}`
`985`		`-returnat::empty_strided(sizes, tensor.strides(), tensor.options());`
	`986`	`+returnat::empty_strided(std::move(sizes), tensor.strides(), tensor.options());`
`986`	`987`	`};`
`987`	`988`
`988`	`989`	`if (grad_input_mask[0]) {`

Comments

(0)