benchmark.cpp
1 #include <iostream> 2 #include <vector> 3 #include <string> 4 #include <filesystem> 5 #include <algorithm> 6 #include <cstdlib> 7 #include <sstream> 8 9 namespace fs = std::filesystem; 10 11 struct TestBinary { 12 std::string compiler; 13 std::string language; 14 std::string implementation; 15 fs::path path; 16 17 std::string getName() const { 18 return compiler + "_" + language + "_" + implementation; 19 } 20 }; 21 22 std::vector<TestBinary> findTestBinaries(const fs::path& testDir, const std::string& testSeries) { 23 std::vector<TestBinary> binaries; 24 25 if (!fs::exists(testDir)) { 26 std::cerr << "Error: Test directory does not exist: " << testDir << std::endl; 27 return binaries; 28 } 29 30 // Iterate through compiler directories (gcc, clang, etc.) 31 for (const auto& compilerEntry : fs::directory_iterator(testDir)) { 32 if (!compilerEntry.is_directory()) continue; 33 34 std::string compiler = compilerEntry.path().filename().string(); 35 36 // Iterate through language directories (cpp, asm, rust, etc.) 37 for (const auto& langEntry : fs::directory_iterator(compilerEntry.path())) { 38 if (!langEntry.is_directory()) continue; 39 40 std::string language = langEntry.path().filename().string(); 41 fs::path seriesPath = langEntry.path() / testSeries; 42 43 if (!fs::exists(seriesPath)) continue; 44 45 // Iterate through implementation directories (001, 002, etc.) 46 for (const auto& implEntry : fs::directory_iterator(seriesPath)) { 47 if (!implEntry.is_directory()) continue; 48 49 std::string implementation = implEntry.path().filename().string(); 50 51 // Find executable in the implementation directory 52 for (const auto& fileEntry : fs::directory_iterator(implEntry.path())) { 53 // Skip .o files and only get executable files 54 if (fileEntry.is_regular_file() && 55 fileEntry.path().extension() != ".o" && 56 (fs::status(fileEntry).permissions() & fs::perms::owner_exec) != fs::perms::none) { 57 58 binaries.push_back({ 59 compiler, 60 language, 61 implementation, 62 fileEntry.path() 63 }); 64 break; // Only take the first executable 65 } 66 } 67 } 68 } 69 } 70 71 // Sort by compiler, language, then implementation 72 std::sort(binaries.begin(), binaries.end(), 73 [](const TestBinary& a, const TestBinary& b) { 74 if (a.compiler != b.compiler) return a.compiler < b.compiler; 75 if (a.language != b.language) return a.language < b.language; 76 return a.implementation < b.implementation; 77 }); 78 79 return binaries; 80 } 81 82 int main(int argc, char* argv[]) { 83 if (argc < 2) { 84 std::cerr << "Usage: " << argv[0] << " <test_series> [hyperfine_options...]" << std::endl; 85 std::cerr << "Example: " << argv[0] << " hello_world" << std::endl; 86 std::cerr << "Example: " << argv[0] << " hello_world --warmup 10 --runs 1000" << std::endl; 87 return 1; 88 } 89 90 std::string testSeries = argv[1]; 91 92 // Get the directory where this executable is located 93 fs::path exePath = fs::canonical("/proc/self/exe"); 94 fs::path binDir = exePath.parent_path(); 95 fs::path projectRoot = binDir.parent_path(); 96 fs::path testDir = projectRoot / "bin" / "tests"; 97 fs::path hyperfinePath = binDir / "hyperfine"; 98 99 // Check if hyperfine exists 100 if (!fs::exists(hyperfinePath)) { 101 std::cerr << "Error: hyperfine not found at: " << hyperfinePath << std::endl; 102 std::cerr << "Please build hyperfine first with: cmake --build build" << std::endl; 103 return 1; 104 } 105 106 std::cout << "==========================================================\n"; 107 std::cout << " Benchmark Series: " << testSeries << "\n"; 108 std::cout << "==========================================================\n\n"; 109 110 // Find all test binaries 111 auto binaries = findTestBinaries(testDir, testSeries); 112 113 if (binaries.empty()) { 114 std::cerr << "Error: No test binaries found for series '" << testSeries << "'" << std::endl; 115 std::cerr << "Searched in: " << testDir << std::endl; 116 return 1; 117 } 118 119 std::cout << "Found " << binaries.size() << " implementations:\n"; 120 for (const auto& bin : binaries) { 121 std::cout << " - " << bin.getName() << ": " << bin.path << "\n"; 122 } 123 std::cout << "\n"; 124 125 // Build hyperfine command 126 std::ostringstream cmd; 127 cmd << hyperfinePath.string(); 128 129 // Add default options 130 cmd << " --warmup 5"; 131 cmd << " --style full"; 132 cmd << " --export-markdown results_" << testSeries << ".md"; 133 cmd << " --export-json results_" << testSeries << ".json"; 134 135 // Add any additional user-provided options (skip argv[0] and argv[1]) 136 for (int i = 2; i < argc; ++i) { 137 cmd << " " << argv[i]; 138 } 139 140 // Add all binaries with their names 141 for (const auto& bin : binaries) { 142 cmd << " --command-name '" << bin.getName() << "'"; 143 cmd << " '" << bin.path.string() << "'"; 144 } 145 146 std::cout << "Running benchmark...\n"; 147 std::cout << "Command: " << cmd.str() << "\n\n"; 148 std::cout << "==========================================================\n\n"; 149 150 // Execute hyperfine 151 int result = std::system(cmd.str().c_str()); 152 153 if (result == 0) { 154 std::cout << "\n==========================================================\n"; 155 std::cout << "Benchmark complete!\n"; 156 std::cout << "Results saved to:\n"; 157 std::cout << " - results_" << testSeries << ".md (Markdown table)\n"; 158 std::cout << " - results_" << testSeries << ".json (JSON data)\n"; 159 std::cout << "==========================================================\n"; 160 } else { 161 std::cerr << "\nError: Benchmark failed with exit code " << result << std::endl; 162 return 1; 163 } 164 165 return 0; 166 }