Flux dev full fine tune config file

[Anothergazz]

I post this here as it may help some other poor soul, who is trying to perform a 'full fine tune/dreambooth' on flux dev. It took me a long time to get it to work. I have an RTX4090 with 64G RAM. I would suggest you really need a min of 64G and more would be helpful, as you can see it thrashing the C: drive as it uses the virtual memory, when writing out the resulting tuned file.
I could not find a way to get it to fine tune an already tuned fluxdev, other than replacing the flux1-dev.safetensors in the Folder were base model lives. Any suggestiions?
Complete new starters go here first
https://www.reddit.com/r/StableDiffusion/comments/1f93un3/onetrainer_flux_training_setup_mystery_solved/?rdt=39307

Any suggestions improvements are welcome, and thank you to the the team behind Onetrainer.

{
"__version": 6,
"training_method": "FINE_TUNE",
"model_type": "FLUX_DEV_1",
"debug_mode": false,
"debug_dir": "debug",
"workspace_dir": "path to the workspace folder",
"cache_dir": "path to the workspace cache folder",
"tensorboard": true,
"tensorboard_expose": false,
"tensorboard_port": 6006,
"validation": false,
"validate_after": 1,
"validate_after_unit": "EPOCH",
"continue_last_backup": true,
"include_train_config": "NONE",
"base_model_name": "Folder were base model++ lives",
"weight_dtype": "FLOAT_16",
"output_dtype": "FLOAT_16",
"output_model_format": "SAFETENSORS",
"output_model_destination": "Save Path and name of tune base model",
"gradient_checkpointing": "CPU_OFFLOADED",
"enable_async_offloading": true,
"enable_activation_offloading": true,
"layer_offload_fraction": 0.3,
"force_circular_padding": false,
"concept_file_name": "training_concepts/concepts.json",
"concepts": null,
"aspect_ratio_bucketing": true,
"latent_caching": true,
"clear_cache_before_training": false,
"learning_rate_scheduler": "CONSTANT",
"custom_learning_rate_scheduler": null,
"scheduler_params": [],
"learning_rate": 3e-05,
"learning_rate_warmup_steps": 200.0,
"learning_rate_cycles": 1.0,
"learning_rate_min_factor": 0.0,
"epochs": 400,
"batch_size": 1,
"gradient_accumulation_steps": 1,
"ema": "OFF",
"ema_decay": 0.999,
"ema_update_step_interval": 5,
"dataloader_threads": 1,
"train_device": "cuda",
"temp_device": "cpu",
"train_dtype": "BFLOAT_16",
"fallback_train_dtype": "BFLOAT_16",
"enable_autocast_cache": true,
"only_cache": false,
"resolution": "1024",
"attention_mechanism": "XFORMERS",
"align_prop": false,
"align_prop_probability": 0.1,
"align_prop_loss": "AESTHETIC",
"align_prop_weight": 0.01,
"align_prop_steps": 20,
"align_prop_truncate_steps": 0.5,
"align_prop_cfg_scale": 7.0,
"mse_strength": 1.0,
"mae_strength": 0.0,
"log_cosh_strength": 0.0,
"vb_loss_strength": 1.0,
"loss_weight_fn": "CONSTANT",
"loss_weight_strength": 5.0,
"dropout_probability": 0.0,
"loss_scaler": "NONE",
"learning_rate_scaler": "NONE",
"clip_grad_norm": 1.0,
"offset_noise_weight": 0.0,
"perturbation_noise_weight": 0.0,
"rescale_noise_scheduler_to_zero_terminal_snr": false,
"force_v_prediction": false,
"force_epsilon_prediction": false,
"min_noising_strength": 0.0,
"max_noising_strength": 1.0,
"timestep_distribution": "LOGIT_NORMAL",
"noising_weight": 0.0,
"noising_bias": 0.0,
"unet": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 0,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"prior": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 0,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"text_encoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": false,
"stop_training_after": 30,
"stop_training_after_unit": "EPOCH",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"text_encoder_layer_skip": 0,
"text_encoder_2": {
"__version": 0,
"model_name": "",
"include": true,
"train": false,
"stop_training_after": 30,
"stop_training_after_unit": "EPOCH",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"text_encoder_2_layer_skip": 0,
"text_encoder_3": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 30,
"stop_training_after_unit": "EPOCH",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"text_encoder_3_layer_skip": 0,
"vae": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"effnet_encoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"decoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"decoder_text_encoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"decoder_vqgan": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false,
"guidance_scale": 1.0
},
"masked_training": false,
"unmasked_probability": 0.1,
"unmasked_weight": 0.1,
"normalize_masked_area_loss": false,
"embedding_learning_rate": null,
"preserve_embedding_norm": false,
"embedding": {
"__version": 0,
"uuid": "48f86ce5-c075-49f2-9edd-802c7ffd24bf",
"model_name": "",
"placeholder": "",
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"token_count": 1,
"initial_embedding_text": "*"
},
"additional_embeddings": [],
"embedding_weight_dtype": "FLOAT_32",
"cloud": {
"__version": 0,
"enabled": false,
"type": "RUNPOD",
"file_sync": "NATIVE_SCP",
"create": true,
"name": "OneTrainer",
"tensorboard_tunnel": true,
"sub_type": "",
"gpu_type": "",
"volume_size": 100,
"min_download": 0,
"remote_dir": "/workspace",
"huggingface_cache_dir": "/workspace/huggingface_cache",
"onetrainer_dir": "/workspace/OneTrainer",
"install_cmd": "git clone https://github.com/Nerogar/OneTrainer",
"install_onetrainer": true,
"update_onetrainer": true,
"detach_trainer": false,
"run_id": "job1",
"download_samples": true,
"download_output_model": true,
"download_saves": true,
"download_backups": false,
"download_tensorboard": false,
"delete_workspace": false,
"on_finish": "NONE",
"on_error": "NONE",
"on_detached_finish": "NONE",
"on_detached_error": "NONE"
},
"peft_type": "LORA",
"lora_model_name": "Path of base model/flux1-dev.safetensors",
"lora_rank": 124,
"lora_alpha": 1.0,
"lora_decompose": false,
"lora_decompose_norm_epsilon": true,
"lora_weight_dtype": "FLOAT_32",
"lora_layers": "",
"lora_layer_preset": null,
"bundle_additional_embeddings": true,
"optimizer": {
"__version": 0,
"optimizer": "ADAFACTOR",
"adam_w_mode": false,
"alpha": null,
"amsgrad": false,
"beta1": null,
"beta2": null,
"beta3": null,
"bias_correction": false,
"block_wise": false,
"capturable": false,
"centered": false,
"clip_threshold": 1.0,
"d0": null,
"d_coef": null,
"dampening": null,
"decay_rate": -0.8,
"decouple": false,
"differentiable": false,
"eps": 1e-30,
"eps2": 0.001,
"foreach": false,
"fsdp_in_use": false,
"fused": false,
"fused_back_pass": true,
"growth_rate": null,
"initial_accumulator_value": null,
"is_paged": false,
"log_every": null,
"lr_decay": null,
"max_unorm": null,
"maximize": false,
"min_8bit_size": null,
"momentum": null,
"nesterov": false,
"no_prox": false,
"optim_bits": null,
"percentile_clipping": null,
"r": null,
"relative_step": false,
"safeguard_warmup": false,
"scale_parameter": false,
"stochastic_rounding": true,
"use_bias_correction": false,
"use_triton": false,
"warmup_init": false,
"weight_decay": 0.0,
"weight_lr_power": null,
"decoupled_decay": false,
"fixed_decay": false,
"rectify": false,
"degenerated_to_sgd": false,
"k": null,
"xi": null,
"n_sma_threshold": null,
"ams_bound": false,
"adanorm": false,
"adam_debias": false,
"slice_p": null,
"cautious": false
},
"optimizer_defaults": {
"ADAMW": {
"__version": 0,
"optimizer": "ADAMW",
"adam_w_mode": false,
"alpha": null,
"amsgrad": false,
"beta1": 0.9,
"beta2": 0.999,
"beta3": null,
"bias_correction": false,
"block_wise": false,
"capturable": false,
"centered": false,
"clip_threshold": null,
"d0": null,
"d_coef": null,
"dampening": null,
"decay_rate": null,
"decouple": false,
"differentiable": false,
"eps": 1e-08,
"eps2": null,
"foreach": false,
"fsdp_in_use": false,
"fused": true,
"fused_back_pass": true,
"growth_rate": null,
"initial_accumulator_value": null,
"is_paged": false,
"log_every": null,
"lr_decay": null,
"max_unorm": null,
"maximize": false,
"min_8bit_size": null,
"momentum": null,
"nesterov": false,
"no_prox": false,
"optim_bits": null,
"percentile_clipping": null,
"r": null,
"relative_step": false,
"safeguard_warmup": false,
"scale_parameter": false,
"stochastic_rounding": false,
"use_bias_correction": false,
"use_triton": false,
"warmup_init": false,
"weight_decay": 0.01,
"weight_lr_power": null,
"decoupled_decay": false,
"fixed_decay": false,
"rectify": false,
"degenerated_to_sgd": false,
"k": null,
"xi": null,
"n_sma_threshold": null,
"ams_bound": false,
"adanorm": false,
"adam_debias": false,
"slice_p": null,
"cautious": false
},
"ADAFACTOR": {
"__version": 0,
"optimizer": "ADAFACTOR",
"adam_w_mode": false,
"alpha": null,
"amsgrad": false,
"beta1": null,
"beta2": null,
"beta3": null,
"bias_correction": false,
"block_wise": false,
"capturable": false,
"centered": false,
"clip_threshold": 1.0,
"d0": null,
"d_coef": null,
"dampening": null,
"decay_rate": -0.8,
"decouple": false,
"differentiable": false,
"eps": 1e-30,
"eps2": 0.001,
"foreach": false,
"fsdp_in_use": false,
"fused": false,
"fused_back_pass": true,
"growth_rate": null,
"initial_accumulator_value": null,
"is_paged": false,
"log_every": null,
"lr_decay": null,
"max_unorm": null,
"maximize": false,
"min_8bit_size": null,
"momentum": null,
"nesterov": false,
"no_prox": false,
"optim_bits": null,
"percentile_clipping": null,
"r": null,
"relative_step": false,
"safeguard_warmup": false,
"scale_parameter": false,
"stochastic_rounding": true,
"use_bias_correction": false,
"use_triton": false,
"warmup_init": false,
"weight_decay": 0.0,
"weight_lr_power": null,
"decoupled_decay": false,
"fixed_decay": false,
"rectify": false,
"degenerated_to_sgd": false,
"k": null,
"xi": null,
"n_sma_threshold": null,
"ams_bound": false,
"adanorm": false,
"adam_debias": false,
"slice_p": null,
"cautious": false
}
},
"sample_definition_file_name": "training_samples/samples.json",
"samples": null,
"sample_after": 60,
"sample_after_unit": "MINUTE",
"sample_image_format": "JPG",
"sample_video_format": "MP4",
"sample_audio_format": "MP3",
"samples_to_tensorboard": true,
"non_ema_sampling": true,
"backup_after": 40,
"backup_after_unit": "MINUTE",
"rolling_backup": true,
"rolling_backup_count": 5,
"backup_before_save": false,
"save_every": 0,
"save_every_unit": "NEVER",
"save_skip_first": 1,
"save_filename_prefix": ""
}