Runtime Error while running local sample notebook #224
Description
I get this error when I try to render frames. How do I solve this?
RuntimeError Traceback (most recent call last)
Cell In [5], line 35
32 for c_o in CONFIG_OVERRIDES:
33 cfg = compose(config_name=CONFIG_DEFAULTS,
34 overrides=[f"conf={c_o}"])
---> 35 render_frames(cfg)
File ~\anaconda3\envs\pytti-tools\lib\site-packages\hydra\main.py:79, in main..main_decorator..decorated_main(cfg_passthrough)
76 @functools.wraps(task_function)
77 def decorated_main(cfg_passthrough: Optional[DictConfig] = None) -> Any:
78 if cfg_passthrough is not None:
---> 79 return task_function(cfg_passthrough)
80 else:
81 args_parser = get_args_parser()
File ~\anaconda3\envs\pytti-tools\lib\site-packages\pytti\workhorse.py:622, in _main(cfg)
620 torch.cuda.empty_cache()
621 else:
--> 622 do_run()
623 logger.info("Complete.")
624 gc.collect()
File ~\anaconda3\envs\pytti-tools\lib\site-packages\pytti\workhorse.py:569, in _main..do_run()
567 for scene in prompts[skip_prompts:]:
568 logger.info("Running prompt:", " | ".join(map(str, scene)))
--> 569 i += model.run_steps(
570 params.steps_per_scene - skip_steps,
571 scene,
572 last_scene,
573 loss_augs,
574 interp_steps=params.interpolation_steps,
575 i_offset=i,
576 skipped_steps=skip_steps,
577 gradient_accumulation_steps=params.gradient_accumulation_steps,
578 )
579 skip_steps = 0
580 model.clear_dataframe()
File ~\anaconda3\envs\pytti-tools\lib\site-packages\pytti\ImageGuide.py:188, in DirectImageGuide.run_steps(self, n_steps, prompts, interp_prompts, loss_augs, stop, interp_steps, i_offset, skipped_steps, gradient_accumulation_steps)
160 for i in tqdm(range(n_steps)):
161 # not a huge fan of this.
162 # currently need it for PixelImage.encode_image
(...)
168 # if not self.null_update:
169 # self.update(i + i_offset, i + skipped_steps)
170 self.update(
171 model=self,
172 img=self.image_rep,
(...)
186 semantic_init_prompt=self.semantic_init_prompt,
187 )
--> 188 losses = self.train(
189 i + skipped_steps,
190 prompts,
191 interp_prompts,
192 loss_augs,
193 interp_steps=interp_steps,
194 gradient_accumulation_steps=gradient_accumulation_steps,
195 )
196 if losses["TOTAL"] <= stop:
197 break
File ~\anaconda3\envs\pytti-tools\lib\site-packages\pytti\ImageGuide.py:347, in DirectImageGuide.train(self, i, prompts, interp_prompts, loss_augs, interp_steps, save_loss, gradient_accumulation_steps)
344 total_loss_mb /= gradient_accumulation_steps
346 # total_loss_mb.backward()
--> 347 total_loss_mb.backward(retain_graph=True)
348 # total_loss += total_loss_mb # this is causing it to break
349 # total_loss = total_loss_mb
350
351 # losses = [{k:v} for k,v in losses_accumulator.items()]
352 # losses_raw = [{k:v} for k,v in losses_raw_accumulator.items()]
353 losses_raw.append({"TOTAL": total_loss}) # this needs to be fixed
File ~\anaconda3\envs\pytti-tools\lib\site-packages\torch_tensor.py:487, in Tensor.backward(self, gradient, retain_graph, create_graph, inputs)
477 if has_torch_function_unary(self):
478 return handle_torch_function(
479 Tensor.backward,
480 (self,),
(...)
485 inputs=inputs,
486 )
--> 487 torch.autograd.backward(
488 self, gradient, retain_graph, create_graph, inputs=inputs
489 )
File ~\anaconda3\envs\pytti-tools\lib\site-packages\torch\autograd_init_.py:197, in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
192 retain_graph = create_graph
194 # The reason we repeat same the comment below is that
195 # some Python versions print out the first line of a multi-line function
196 # calls in the traceback and some print out the last line
--> 197 Variable.execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
198 tensors, grad_tensors, retain_graph, create_graph, inputs,
199 allow_unreachable=True, accumulate_grad=True)
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [40, 1]], which is output 0 of NormBackward1, is at version 4; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).