Benchmark GPU Performance

runcrate instances create --name bench-4090 --gpu RTX4090
runcrate instances create --name bench-a100 --gpu A100
runcrate instances create --name bench-h100 --gpu H100

runcrate instances status bench-4090
runcrate instances status bench-a100
runcrate instances status bench-h100

runcrate ssh bench-4090 -- "pip install torch vllm"
runcrate ssh bench-a100 -- "pip install torch vllm"
runcrate ssh bench-h100 -- "pip install torch vllm"

runcrate ssh bench-4090 -- "python -c \"
import torch, time
size = 8192
a = torch.randn(size, size, dtype=torch.float16, device='cuda')
b = torch.randn(size, size, dtype=torch.float16, device='cuda')
for _ in range(10): torch.mm(a, b)
torch.cuda.synchronize()
start = time.time()
for _ in range(100): torch.mm(a, b)
torch.cuda.synchronize()
elapsed = time.time() - start
tflops = (2 * size**3 * 100) / elapsed / 1e12
print(f'FP16 matmul: {tflops:.1f} TFLOPS ({elapsed:.2f}s for 100 iters)')
\""

runcrate ssh bench-4090 -- "python -c \"
import torch, time
size = 256 * 1024 * 1024
a = torch.randn(size, dtype=torch.float16, device='cuda')
b = torch.empty_like(a)
torch.cuda.synchronize()
start = time.time()
for _ in range(100): b.copy_(a)
torch.cuda.synchronize()
elapsed = time.time() - start
bw = (2 * size * 2 * 100) / elapsed / 1e9
print(f'Memory bandwidth: {bw:.0f} GB/s')
\""

runcrate ssh bench-4090 -- "python -c \"
import torch
p = torch.cuda.get_device_properties(0)
print(f'GPU: {p.name}, VRAM: {p.total_mem / 1e9:.1f} GB, SMs: {p.multi_processor_count}')
\""

runcrate instances delete bench-4090
runcrate instances delete bench-a100
runcrate instances delete bench-h100