1  #[cfg(windows)]
  2  use std::os::windows::process::CommandExt;
  3  use std::process::ExitStatus;
  4  
  5  use crate::command::Command;
  6  use crate::options::{
  7      CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
  8  };
  9  use crate::output::progress_bar::get_progress_bar;
 10  use crate::timer::{execute_and_measure, TimerResult};
 11  use crate::util::randomized_environment_offset;
 12  use crate::util::units::Second;
 13  
 14  use super::timing_result::TimingResult;
 15  
 16  use anyhow::{bail, Context, Result};
 17  use statistical::mean;
 18  
 19  pub enum BenchmarkIteration {
 20      NonBenchmarkRun,
 21      Warmup(u64),
 22      Benchmark(u64),
 23  }
 24  
 25  impl BenchmarkIteration {
 26      pub fn to_env_var_value(&self) -> Option<String> {
 27          match self {
 28              BenchmarkIteration::NonBenchmarkRun => None,
 29              BenchmarkIteration::Warmup(i) => Some(format!("warmup-{}", i)),
 30              BenchmarkIteration::Benchmark(i) => Some(format!("{}", i)),
 31          }
 32      }
 33  }
 34  
 35  pub trait Executor {
 36      /// Run the given command and measure the execution time
 37      fn run_command_and_measure(
 38          &self,
 39          command: &Command<'_>,
 40          iteration: BenchmarkIteration,
 41          command_failure_action: Option<CmdFailureAction>,
 42          output_policy: &CommandOutputPolicy,
 43      ) -> Result<(TimingResult, ExitStatus)>;
 44  
 45      /// Perform a calibration of this executor. For example,
 46      /// when running commands through a shell, we need to
 47      /// measure the shell spawning time separately in order
 48      /// to subtract it from the full runtime later.
 49      fn calibrate(&mut self) -> Result<()>;
 50  
 51      /// Return the time overhead for this executor when
 52      /// performing a measurement. This should return the time
 53      /// that is being used in addition to the actual runtime
 54      /// of the command.
 55      fn time_overhead(&self) -> Second;
 56  }
 57  
 58  fn run_command_and_measure_common(
 59      mut command: std::process::Command,
 60      iteration: BenchmarkIteration,
 61      command_failure_action: CmdFailureAction,
 62      command_input_policy: &CommandInputPolicy,
 63      command_output_policy: &CommandOutputPolicy,
 64      command_name: &str,
 65  ) -> Result<TimerResult> {
 66      let stdin = command_input_policy.get_stdin()?;
 67      let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
 68      command.stdin(stdin).stdout(stdout).stderr(stderr);
 69  
 70      command.env(
 71          "HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET",
 72          randomized_environment_offset::value(),
 73      );
 74  
 75      if let Some(value) = iteration.to_env_var_value() {
 76          command.env("HYPERFINE_ITERATION", value);
 77      }
 78  
 79      let result = execute_and_measure(command)
 80          .with_context(|| format!("Failed to run command '{command_name}'"))?;
 81  
 82      if !result.status.success() {
 83          use crate::util::exit_code::extract_exit_code;
 84  
 85          let should_fail = match command_failure_action {
 86              CmdFailureAction::RaiseError => true,
 87              CmdFailureAction::IgnoreAllFailures => false,
 88              CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
 89                  // Only fail if the exit code is not in the list of codes to ignore
 90                  if let Some(exit_code) = extract_exit_code(result.status) {
 91                      !codes.contains(&exit_code)
 92                  } else {
 93                      // If we can't extract an exit code, treat it as a failure
 94                      true
 95                  }
 96              }
 97          };
 98  
 99          if should_fail {
100              let when = match iteration {
101                  BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
102                  BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
103                  BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
104                  BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
105                  BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
106              };
107              bail!(
108                  "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
109                  Alternatively, use the '--show-output' option to debug what went wrong.",
110                  cause=result.status.code().map_or(
111                      "The process has been terminated by a signal".into(),
112                      |c| format!("Command terminated with non-zero exit code {c}")
113  
114                  ),
115              );
116          }
117      }
118  
119      Ok(result)
120  }
121  
122  pub struct RawExecutor<'a> {
123      options: &'a Options,
124  }
125  
126  impl<'a> RawExecutor<'a> {
127      pub fn new(options: &'a Options) -> Self {
128          RawExecutor { options }
129      }
130  }
131  
132  impl Executor for RawExecutor<'_> {
133      fn run_command_and_measure(
134          &self,
135          command: &Command<'_>,
136          iteration: BenchmarkIteration,
137          command_failure_action: Option<CmdFailureAction>,
138          output_policy: &CommandOutputPolicy,
139      ) -> Result<(TimingResult, ExitStatus)> {
140          let result = run_command_and_measure_common(
141              command.get_command()?,
142              iteration,
143              command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
144              &self.options.command_input_policy,
145              output_policy,
146              &command.get_command_line(),
147          )?;
148  
149          Ok((
150              TimingResult {
151                  time_real: result.time_real,
152                  time_user: result.time_user,
153                  time_system: result.time_system,
154                  memory_usage_byte: result.memory_usage_byte,
155              },
156              result.status,
157          ))
158      }
159  
160      fn calibrate(&mut self) -> Result<()> {
161          Ok(())
162      }
163  
164      fn time_overhead(&self) -> Second {
165          0.0
166      }
167  }
168  
169  pub struct ShellExecutor<'a> {
170      options: &'a Options,
171      shell: &'a Shell,
172      shell_spawning_time: Option<TimingResult>,
173  }
174  
175  impl<'a> ShellExecutor<'a> {
176      pub fn new(shell: &'a Shell, options: &'a Options) -> Self {
177          ShellExecutor {
178              shell,
179              options,
180              shell_spawning_time: None,
181          }
182      }
183  }
184  
185  impl Executor for ShellExecutor<'_> {
186      fn run_command_and_measure(
187          &self,
188          command: &Command<'_>,
189          iteration: BenchmarkIteration,
190          command_failure_action: Option<CmdFailureAction>,
191          output_policy: &CommandOutputPolicy,
192      ) -> Result<(TimingResult, ExitStatus)> {
193          let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
194          let mut command_builder = self.shell.command();
195          command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
196  
197          // Windows needs special treatment for its behavior on parsing cmd arguments
198          if on_windows_cmd {
199              #[cfg(windows)]
200              command_builder.raw_arg(command.get_command_line());
201          } else {
202              command_builder.arg(command.get_command_line());
203          }
204  
205          let mut result = run_command_and_measure_common(
206              command_builder,
207              iteration,
208              command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
209              &self.options.command_input_policy,
210              output_policy,
211              &command.get_command_line(),
212          )?;
213  
214          // Subtract shell spawning time
215          if let Some(spawning_time) = self.shell_spawning_time {
216              result.time_real = (result.time_real - spawning_time.time_real).max(0.0);
217              result.time_user = (result.time_user - spawning_time.time_user).max(0.0);
218              result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
219          }
220  
221          Ok((
222              TimingResult {
223                  time_real: result.time_real,
224                  time_user: result.time_user,
225                  time_system: result.time_system,
226                  memory_usage_byte: result.memory_usage_byte,
227              },
228              result.status,
229          ))
230      }
231  
232      /// Measure the average shell spawning time
233      fn calibrate(&mut self) -> Result<()> {
234          const COUNT: u64 = 50;
235          let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
236              Some(get_progress_bar(
237                  COUNT,
238                  "Measuring shell spawning time",
239                  self.options.output_style,
240              ))
241          } else {
242              None
243          };
244  
245          let mut times_real: Vec<Second> = vec![];
246          let mut times_user: Vec<Second> = vec![];
247          let mut times_system: Vec<Second> = vec![];
248  
249          for _ in 0..COUNT {
250              // Just run the shell without any command
251              let res = self.run_command_and_measure(
252                  &Command::new(None, ""),
253                  BenchmarkIteration::NonBenchmarkRun,
254                  None,
255                  &CommandOutputPolicy::Null,
256              );
257  
258              match res {
259                  Err(_) => {
260                      let shell_cmd = if cfg!(windows) {
261                          format!("{} /C \"\"", self.shell)
262                      } else {
263                          format!("{} -c \"\"", self.shell)
264                      };
265  
266                      bail!(
267                          "Could not measure shell execution time. Make sure you can run '{}'.",
268                          shell_cmd
269                      );
270                  }
271                  Ok((r, _)) => {
272                      times_real.push(r.time_real);
273                      times_user.push(r.time_user);
274                      times_system.push(r.time_system);
275                  }
276              }
277  
278              if let Some(bar) = progress_bar.as_ref() {
279                  bar.inc(1)
280              }
281          }
282  
283          if let Some(bar) = progress_bar.as_ref() {
284              bar.finish_and_clear()
285          }
286  
287          self.shell_spawning_time = Some(TimingResult {
288              time_real: mean(&times_real),
289              time_user: mean(&times_user),
290              time_system: mean(&times_system),
291              memory_usage_byte: 0,
292          });
293  
294          Ok(())
295      }
296  
297      fn time_overhead(&self) -> Second {
298          self.shell_spawning_time.unwrap().time_real
299      }
300  }
301  
302  #[derive(Clone)]
303  pub struct MockExecutor {
304      shell: Option<String>,
305  }
306  
307  impl MockExecutor {
308      pub fn new(shell: Option<String>) -> Self {
309          MockExecutor { shell }
310      }
311  
312      fn extract_time<S: AsRef<str>>(sleep_command: S) -> Second {
313          assert!(sleep_command.as_ref().starts_with("sleep "));
314          sleep_command
315              .as_ref()
316              .trim_start_matches("sleep ")
317              .parse::<Second>()
318              .unwrap()
319      }
320  }
321  
322  impl Executor for MockExecutor {
323      fn run_command_and_measure(
324          &self,
325          command: &Command<'_>,
326          _iteration: BenchmarkIteration,
327          _command_failure_action: Option<CmdFailureAction>,
328          _output_policy: &CommandOutputPolicy,
329      ) -> Result<(TimingResult, ExitStatus)> {
330          #[cfg(unix)]
331          let status = {
332              use std::os::unix::process::ExitStatusExt;
333              ExitStatus::from_raw(0)
334          };
335  
336          #[cfg(windows)]
337          let status = {
338              use std::os::windows::process::ExitStatusExt;
339              ExitStatus::from_raw(0)
340          };
341  
342          Ok((
343              TimingResult {
344                  time_real: Self::extract_time(command.get_command_line()),
345                  time_user: 0.0,
346                  time_system: 0.0,
347                  memory_usage_byte: 0,
348              },
349              status,
350          ))
351      }
352  
353      fn calibrate(&mut self) -> Result<()> {
354          Ok(())
355      }
356  
357      fn time_overhead(&self) -> Second {
358          match &self.shell {
359              None => 0.0,
360              Some(shell) => Self::extract_time(shell),
361          }
362      }
363  }
364  
365  #[test]
366  fn test_mock_executor_extract_time() {
367      assert_eq!(MockExecutor::extract_time("sleep 0.1"), 0.1);
368  }