1  use std::{fs::File, io::Read, path::PathBuf};
  2  
  3  use tempfile::{tempdir, TempDir};
  4  
  5  mod common;
  6  use common::hyperfine;
  7  
  8  struct ExecutionOrderTest {
  9      cmd: assert_cmd::Command,
 10      expected_content: String,
 11      logfile_path: PathBuf,
 12      #[allow(dead_code)]
 13      tempdir: TempDir,
 14  }
 15  
 16  impl ExecutionOrderTest {
 17      fn new() -> Self {
 18          let tempdir = tempdir().unwrap();
 19          let logfile_path = tempdir.path().join("output.log");
 20  
 21          ExecutionOrderTest {
 22              cmd: hyperfine(),
 23              expected_content: String::new(),
 24              logfile_path,
 25              tempdir,
 26          }
 27      }
 28  
 29      fn arg<S: AsRef<str>>(&mut self, arg: S) -> &mut Self {
 30          self.cmd.arg(arg.as_ref());
 31          self
 32      }
 33  
 34      fn get_command(&self, output: &str) -> String {
 35          format!(
 36              "echo {output} >> {path}",
 37              output = output,
 38              path = self.logfile_path.to_string_lossy()
 39          )
 40      }
 41  
 42      fn command(&mut self, output: &str) -> &mut Self {
 43          self.arg(self.get_command(output));
 44          self
 45      }
 46  
 47      fn setup(&mut self, output: &str) -> &mut Self {
 48          self.arg("--setup");
 49          self.command(output)
 50      }
 51  
 52      fn prepare(&mut self, output: &str) -> &mut Self {
 53          self.arg("--prepare");
 54          self.command(output)
 55      }
 56  
 57      fn reference(&mut self, output: &str) -> &mut Self {
 58          self.arg("--reference");
 59          self.command(output)
 60      }
 61  
 62      fn conclude(&mut self, output: &str) -> &mut Self {
 63          self.arg("--conclude");
 64          self.command(output)
 65      }
 66  
 67      fn cleanup(&mut self, output: &str) -> &mut Self {
 68          self.arg("--cleanup");
 69          self.command(output)
 70      }
 71  
 72      fn expect_output(&mut self, output: &str) -> &mut Self {
 73          self.expected_content.push_str(output);
 74  
 75          #[cfg(windows)]
 76          {
 77              self.expected_content.push_str(" \r");
 78          }
 79  
 80          self.expected_content.push('\n');
 81          self
 82      }
 83  
 84      fn run(&mut self) {
 85          self.cmd.assert().success();
 86  
 87          let mut f = File::open(&self.logfile_path).unwrap();
 88          let mut content = String::new();
 89          f.read_to_string(&mut content).unwrap();
 90  
 91          assert_eq!(content, self.expected_content);
 92      }
 93  }
 94  
 95  impl Default for ExecutionOrderTest {
 96      fn default() -> Self {
 97          Self::new()
 98      }
 99  }
100  
101  #[test]
102  fn benchmarks_are_executed_sequentially_one() {
103      ExecutionOrderTest::new()
104          .arg("--runs=1")
105          .command("command 1")
106          .command("command 2")
107          .expect_output("command 1")
108          .expect_output("command 2")
109          .run();
110  }
111  
112  #[test]
113  fn benchmarks_are_executed_sequentially() {
114      ExecutionOrderTest::new()
115          .arg("--runs=2")
116          .command("command 1")
117          .command("command 2")
118          .expect_output("command 1")
119          .expect_output("command 1")
120          .expect_output("command 2")
121          .expect_output("command 2")
122          .run();
123  }
124  
125  #[test]
126  fn warmup_runs_are_executed_before_benchmarking_runs() {
127      ExecutionOrderTest::new()
128          .arg("--runs=2")
129          .arg("--warmup=3")
130          .command("command 1")
131          .expect_output("command 1")
132          .expect_output("command 1")
133          .expect_output("command 1")
134          .expect_output("command 1")
135          .expect_output("command 1")
136          .run();
137  }
138  
139  #[test]
140  fn setup_commands_are_executed_before_each_series_of_timing_runs() {
141      ExecutionOrderTest::new()
142          .arg("--runs=2")
143          .setup("setup")
144          .command("command 1")
145          .command("command 2")
146          .expect_output("setup")
147          .expect_output("command 1")
148          .expect_output("command 1")
149          .expect_output("setup")
150          .expect_output("command 2")
151          .expect_output("command 2")
152          .run();
153  }
154  
155  #[test]
156  fn prepare_commands_are_executed_before_each_timing_run() {
157      ExecutionOrderTest::new()
158          .arg("--runs=2")
159          .prepare("prepare")
160          .command("command 1")
161          .command("command 2")
162          .expect_output("prepare")
163          .expect_output("command 1")
164          .expect_output("prepare")
165          .expect_output("command 1")
166          .expect_output("prepare")
167          .expect_output("command 2")
168          .expect_output("prepare")
169          .expect_output("command 2")
170          .run();
171  }
172  
173  #[test]
174  fn conclude_commands_are_executed_after_each_timing_run() {
175      ExecutionOrderTest::new()
176          .arg("--runs=2")
177          .conclude("conclude")
178          .command("command 1")
179          .command("command 2")
180          .expect_output("command 1")
181          .expect_output("conclude")
182          .expect_output("command 1")
183          .expect_output("conclude")
184          .expect_output("command 2")
185          .expect_output("conclude")
186          .expect_output("command 2")
187          .expect_output("conclude")
188          .run();
189  }
190  
191  #[test]
192  fn prepare_commands_are_executed_before_each_warmup() {
193      ExecutionOrderTest::new()
194          .arg("--warmup=2")
195          .arg("--runs=1")
196          .prepare("prepare")
197          .command("command 1")
198          .command("command 2")
199          // warmup 1
200          .expect_output("prepare")
201          .expect_output("command 1")
202          .expect_output("prepare")
203          .expect_output("command 1")
204          // benchmark 1
205          .expect_output("prepare")
206          .expect_output("command 1")
207          // warmup 2
208          .expect_output("prepare")
209          .expect_output("command 2")
210          .expect_output("prepare")
211          .expect_output("command 2")
212          // benchmark 2
213          .expect_output("prepare")
214          .expect_output("command 2")
215          .run();
216  }
217  
218  #[test]
219  fn conclude_commands_are_executed_after_each_warmup() {
220      ExecutionOrderTest::new()
221          .arg("--warmup=2")
222          .arg("--runs=1")
223          .conclude("conclude")
224          .command("command 1")
225          .command("command 2")
226          // warmup 1
227          .expect_output("command 1")
228          .expect_output("conclude")
229          .expect_output("command 1")
230          .expect_output("conclude")
231          // benchmark 1
232          .expect_output("command 1")
233          .expect_output("conclude")
234          // warmup 2
235          .expect_output("command 2")
236          .expect_output("conclude")
237          .expect_output("command 2")
238          .expect_output("conclude")
239          // benchmark 2
240          .expect_output("command 2")
241          .expect_output("conclude")
242          .run();
243  }
244  
245  #[test]
246  fn cleanup_commands_are_executed_once_after_each_benchmark() {
247      ExecutionOrderTest::new()
248          .arg("--runs=2")
249          .cleanup("cleanup")
250          .command("command 1")
251          .command("command 2")
252          .expect_output("command 1")
253          .expect_output("command 1")
254          .expect_output("cleanup")
255          .expect_output("command 2")
256          .expect_output("command 2")
257          .expect_output("cleanup")
258          .run();
259  }
260  
261  #[test]
262  fn setup_prepare_cleanup_combined() {
263      ExecutionOrderTest::new()
264          .arg("--warmup=1")
265          .arg("--runs=2")
266          .setup("setup")
267          .prepare("prepare")
268          .command("command1")
269          .command("command2")
270          .cleanup("cleanup")
271          // 1
272          .expect_output("setup")
273          .expect_output("prepare")
274          .expect_output("command1")
275          .expect_output("prepare")
276          .expect_output("command1")
277          .expect_output("prepare")
278          .expect_output("command1")
279          .expect_output("cleanup")
280          // 2
281          .expect_output("setup")
282          .expect_output("prepare")
283          .expect_output("command2")
284          .expect_output("prepare")
285          .expect_output("command2")
286          .expect_output("prepare")
287          .expect_output("command2")
288          .expect_output("cleanup")
289          .run();
290  }
291  
292  #[test]
293  fn setup_prepare_conclude_cleanup_combined() {
294      ExecutionOrderTest::new()
295          .arg("--warmup=1")
296          .arg("--runs=2")
297          .setup("setup")
298          .prepare("prepare")
299          .command("command1")
300          .command("command2")
301          .conclude("conclude")
302          .cleanup("cleanup")
303          // 1
304          .expect_output("setup")
305          .expect_output("prepare")
306          .expect_output("command1")
307          .expect_output("conclude")
308          .expect_output("prepare")
309          .expect_output("command1")
310          .expect_output("conclude")
311          .expect_output("prepare")
312          .expect_output("command1")
313          .expect_output("conclude")
314          .expect_output("cleanup")
315          // 2
316          .expect_output("setup")
317          .expect_output("prepare")
318          .expect_output("command2")
319          .expect_output("conclude")
320          .expect_output("prepare")
321          .expect_output("command2")
322          .expect_output("conclude")
323          .expect_output("prepare")
324          .expect_output("command2")
325          .expect_output("conclude")
326          .expect_output("cleanup")
327          .run();
328  }
329  
330  #[test]
331  fn single_parameter_value() {
332      ExecutionOrderTest::new()
333          .arg("--runs=2")
334          .arg("--parameter-list")
335          .arg("number")
336          .arg("1,2,3")
337          .command("command {number}")
338          .expect_output("command 1")
339          .expect_output("command 1")
340          .expect_output("command 2")
341          .expect_output("command 2")
342          .expect_output("command 3")
343          .expect_output("command 3")
344          .run();
345  }
346  
347  #[test]
348  fn multiple_parameter_values() {
349      ExecutionOrderTest::new()
350          .arg("--runs=2")
351          .arg("--parameter-list")
352          .arg("number")
353          .arg("1,2,3")
354          .arg("--parameter-list")
355          .arg("letter")
356          .arg("a,b")
357          .command("command {number} {letter}")
358          .expect_output("command 1 a")
359          .expect_output("command 1 a")
360          .expect_output("command 2 a")
361          .expect_output("command 2 a")
362          .expect_output("command 3 a")
363          .expect_output("command 3 a")
364          .expect_output("command 1 b")
365          .expect_output("command 1 b")
366          .expect_output("command 2 b")
367          .expect_output("command 2 b")
368          .expect_output("command 3 b")
369          .expect_output("command 3 b")
370          .run();
371  }
372  
373  #[test]
374  fn reference_is_executed_first() {
375      ExecutionOrderTest::new()
376          .arg("--runs=1")
377          .reference("reference")
378          .command("command 1")
379          .command("command 2")
380          .expect_output("reference")
381          .expect_output("command 1")
382          .expect_output("command 2")
383          .run();
384  }
385  
386  #[test]
387  fn reference_is_executed_first_parameter_value() {
388      ExecutionOrderTest::new()
389          .arg("--runs=2")
390          .reference("reference")
391          .arg("--parameter-list")
392          .arg("number")
393          .arg("1,2,3")
394          .command("command {number}")
395          .expect_output("reference")
396          .expect_output("reference")
397          .expect_output("command 1")
398          .expect_output("command 1")
399          .expect_output("command 2")
400          .expect_output("command 2")
401          .expect_output("command 3")
402          .expect_output("command 3")
403          .run();
404  }
405  
406  #[test]
407  fn reference_is_executed_separately_from_commands() {
408      ExecutionOrderTest::new()
409          .arg("--runs=1")
410          .reference("command 1")
411          .command("command 1")
412          .command("command 2")
413          .expect_output("command 1")
414          .expect_output("command 1")
415          .expect_output("command 2")
416          .run();
417  }
418  
419  #[test]
420  fn setup_prepare_reference_conclude_cleanup_combined() {
421      ExecutionOrderTest::new()
422          .arg("--warmup=1")
423          .arg("--runs=2")
424          .setup("setup")
425          .prepare("prepare")
426          .reference("reference")
427          .command("command1")
428          .command("command2")
429          .conclude("conclude")
430          .cleanup("cleanup")
431          // reference
432          .expect_output("setup")
433          .expect_output("prepare")
434          .expect_output("reference")
435          .expect_output("conclude")
436          .expect_output("prepare")
437          .expect_output("reference")
438          .expect_output("conclude")
439          .expect_output("prepare")
440          .expect_output("reference")
441          .expect_output("conclude")
442          .expect_output("cleanup")
443          // 1
444          .expect_output("setup")
445          .expect_output("prepare")
446          .expect_output("command1")
447          .expect_output("conclude")
448          .expect_output("prepare")
449          .expect_output("command1")
450          .expect_output("conclude")
451          .expect_output("prepare")
452          .expect_output("command1")
453          .expect_output("conclude")
454          .expect_output("cleanup")
455          // 2
456          .expect_output("setup")
457          .expect_output("prepare")
458          .expect_output("command2")
459          .expect_output("conclude")
460          .expect_output("prepare")
461          .expect_output("command2")
462          .expect_output("conclude")
463          .expect_output("prepare")
464          .expect_output("command2")
465          .expect_output("conclude")
466          .expect_output("cleanup")
467          .run();
468  }
469  
470  #[test]
471  fn setup_separate_prepare_separate_conclude_cleanup_combined() {
472      ExecutionOrderTest::new()
473          .arg("--warmup=1")
474          .arg("--runs=2")
475          .setup("setup")
476          .cleanup("cleanup")
477          .prepare("prepare1")
478          .command("command1")
479          .conclude("conclude1")
480          .prepare("prepare2")
481          .command("command2")
482          .conclude("conclude2")
483          // 1
484          .expect_output("setup")
485          .expect_output("prepare1")
486          .expect_output("command1")
487          .expect_output("conclude1")
488          .expect_output("prepare1")
489          .expect_output("command1")
490          .expect_output("conclude1")
491          .expect_output("prepare1")
492          .expect_output("command1")
493          .expect_output("conclude1")
494          .expect_output("cleanup")
495          // 2
496          .expect_output("setup")
497          .expect_output("prepare2")
498          .expect_output("command2")
499          .expect_output("conclude2")
500          .expect_output("prepare2")
501          .expect_output("command2")
502          .expect_output("conclude2")
503          .expect_output("prepare2")
504          .expect_output("command2")
505          .expect_output("conclude2")
506          .expect_output("cleanup")
507          .run();
508  }
509  
510  #[test]
511  fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
512      ExecutionOrderTest::new()
513          .arg("--warmup=1")
514          .arg("--runs=2")
515          .setup("setup")
516          .cleanup("cleanup")
517          .prepare("prepareref")
518          .reference("reference")
519          .conclude("concluderef")
520          .prepare("prepare1")
521          .command("command1")
522          .conclude("conclude1")
523          .prepare("prepare2")
524          .command("command2")
525          .conclude("conclude2")
526          // reference
527          .expect_output("setup")
528          .expect_output("prepareref")
529          .expect_output("reference")
530          .expect_output("concluderef")
531          .expect_output("prepareref")
532          .expect_output("reference")
533          .expect_output("concluderef")
534          .expect_output("prepareref")
535          .expect_output("reference")
536          .expect_output("concluderef")
537          .expect_output("cleanup")
538          // 1
539          .expect_output("setup")
540          .expect_output("prepare1")
541          .expect_output("command1")
542          .expect_output("conclude1")
543          .expect_output("prepare1")
544          .expect_output("command1")
545          .expect_output("conclude1")
546          .expect_output("prepare1")
547          .expect_output("command1")
548          .expect_output("conclude1")
549          .expect_output("cleanup")
550          // 2
551          .expect_output("setup")
552          .expect_output("prepare2")
553          .expect_output("command2")
554          .expect_output("conclude2")
555          .expect_output("prepare2")
556          .expect_output("command2")
557          .expect_output("conclude2")
558          .expect_output("prepare2")
559          .expect_output("command2")
560          .expect_output("conclude2")
561          .expect_output("cleanup")
562          .run();
563  }