I have the following code to train my model. Everything works fine, except it is not logging the vAcc and vLoss to wandb dashboard. The train_accuracy and such are all logged fine and shown, its only the validation which is not. Am i doing something wrong somewhere? I know the validation value is being calculated because i see it being printed out during the loop
import ray
from ray import tune
import time
from ray.air.integrations.wandb import WandbLoggerCallback, setup_wandb
import wandb
import os
os.environ["WANDB_API_KEY"] = "KEY"
wandb.login()
def train_function(config):
wandb = setup_wandb(config)
#train loop
for epoch in range(num_epochs):
total_loss_train += loss.item() * len(batch) # Accumulate the loss
average_loss_train = total_loss_train / len(train_loader.dataset) # Calculate average loss
# Calculate training accuracy
train_accuracy = calculate_accuracy(train_loader, model)
# Switch model to evaluation mode for validation loss and accuracy
model.eval()
total_loss_val = 0.0 # Initialize total loss for validation
total_loss_val += loss.item() * len(batch) # Accumulate the loss
print("Doing validation calculations!")
average_loss_val = total_loss_val / len(val_loader.dataset) # Calculate average loss
wandb.log({"vLoss":average_loss_val})
print(average_loss_val)
# Calculate validation accuracy
val_accuracy = calculate_accuracy(val_loader, model)
wandb.log({"vAcc":val_accuracy})
print(f'Epoch {epoch+1}/{num_epochs}, Training Loss: {average_loss_train:.4f}, Training Accuracy: {train_accuracy:.4f}%, Validation Loss: {average_loss_val:.4f}, Validation Accuracy: {val_accuracy:.4f}%, Epoch time: {epoch_time}s')
wandb.log({"train_loss": average_loss_train,
"train_acc": train_accuracy})
config = {
"lr": tune.loguniform(1e-4, 1e-1), # Learning rate (log scale)
"dropout": tune.uniform(0.0, 0.5), # Dropout rate
"weight_decay": tune.loguniform(1e-6, 1e-3), # L2 regularization (log scale)
"optimizer": tune.choice(["adam", "sgd"]), # Optimizer choice
"num_epochs": tune.choice([10, 20]), # Number of training epochs
}
analysis = tune.run(
train_function,
config=config, # Your hyperparameter configuration
num_samples=1, # Number of trials to run
resources_per_trial={"cpu": 1, "gpu": 1}, # Adjust based on your GPU availability
name="experiment1", # Specify a name for your Ray Tune experiment
callbacks=[WandbLoggerCallback(project="NFL")]
)
wandb.finish()