1  import { execSync } from 'child_process';
  2  import { versions } from './versions';
  3  import yargs from 'yargs';
  4  import fs from 'fs';
  5  import { BenchmarkResults, Benchmark, Result, IperfResults, PingResults, ResultValue } from './benchmark-result-type';
  6  
  7  async function main(clientPublicIP: string, serverPublicIP: string, testing: boolean) {
  8      const pings = runPing(clientPublicIP, serverPublicIP, testing);
  9      const iperf = runIPerf(clientPublicIP, serverPublicIP, testing);
 10  
 11      copyAndBuildPerfImplementations(serverPublicIP);
 12      copyAndBuildPerfImplementations(clientPublicIP);
 13  
 14      const benchmarks = [
 15          runBenchmarkAcrossVersions({
 16              name: "throughput/upload",
 17              clientPublicIP,
 18              serverPublicIP,
 19              uploadBytes: Number.MAX_SAFE_INTEGER,
 20              downloadBytes: 0,
 21              unit: "bit/s",
 22              iterations: testing ? 1 : 10,
 23              durationSecondsPerIteration: testing ? 5 : 20,
 24          }),
 25          runBenchmarkAcrossVersions({
 26              name: "throughput/download",
 27              clientPublicIP,
 28              serverPublicIP,
 29              uploadBytes: 0,
 30              downloadBytes: Number.MAX_SAFE_INTEGER,
 31              unit: "bit/s",
 32              iterations: testing ? 1 : 10,
 33              durationSecondsPerIteration: testing ? 5 : 20,
 34          }),
 35          runBenchmarkAcrossVersions({
 36              name: "Connection establishment + 1 byte round trip latencies",
 37              clientPublicIP,
 38              serverPublicIP,
 39              uploadBytes: 1,
 40              downloadBytes: 1,
 41              unit: "s",
 42              iterations: testing ? 1 : 100,
 43              durationSecondsPerIteration: Number.MAX_SAFE_INTEGER,
 44          }),
 45      ];
 46  
 47      const benchmarkResults: BenchmarkResults = {
 48          benchmarks,
 49          pings,
 50          iperf,
 51      };
 52  
 53      // Save results to benchmark-results.json
 54      fs.writeFileSync('./benchmark-results.json', JSON.stringify(benchmarkResults, null, 2));
 55  
 56      console.error("== done");
 57  }
 58  
 59  function runPing(clientPublicIP: string, serverPublicIP: string, testing: boolean): PingResults {
 60      const pingCount = testing ? 1 : 100;
 61      console.error(`= run ${pingCount} pings from client to server`);
 62  
 63      const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'ping -c ${pingCount} ${serverPublicIP}'`;
 64      const stdout = execCommand(cmd).toString();
 65  
 66      // Extract the time from each ping
 67      const lines = stdout.split('\n');
 68      const times = lines
 69          .map(line => {
 70              const match = line.match(/time=(.*) ms/);
 71              return match ? parseFloat(match[1]) / 1000 : null; // Convert from ms to s
 72          })
 73          .filter((time): time is number => time !== null); // Remove any null values and ensure that array contains only numbers
 74  
 75      return { unit: "s", results: times }
 76  }
 77  
 78  function runIPerf(clientPublicIP: string, serverPublicIP: string, testing: boolean): IperfResults {
 79      const iPerfIterations = testing ? 1 : 60;
 80      console.error(`= run ${iPerfIterations} iPerf TCP from client to server`);
 81  
 82      const killCMD = `ssh -o StrictHostKeyChecking=no ec2-user@${serverPublicIP} 'kill $(cat pidfile); rm pidfile; rm server.log || true'`;
 83      const killSTDOUT = execCommand(killCMD);
 84      console.error(killSTDOUT);
 85  
 86      const serverCMD = `ssh -o StrictHostKeyChecking=no ec2-user@${serverPublicIP} 'nohup iperf3 -s > server.log 2>&1 & echo \$! > pidfile '`;
 87      const serverSTDOUT = execCommand(serverCMD);
 88      console.error(serverSTDOUT);
 89  
 90      const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'iperf3 -c ${serverPublicIP} -t ${iPerfIterations} -N'`;
 91      const stdout = execSync(cmd).toString();
 92  
 93      // Extract the bitrate from each relevant line
 94      const lines = stdout.split('\n');
 95      const bitrates = lines
 96          .map(line => {
 97              const match = line.match(/(\d+(?:\.\d+)?) (\w)bits\/sec/); // Matches and captures the number and unit before "bits/sec"
 98              if (match) {
 99                  const value = parseFloat(match[1]);
100                  const unit = match[2];
101                  // Convert value to bits/sec
102                  const multiplier = unit === 'G' ? 1e9 : unit === 'M' ? 1e6 : unit === 'K' ? 1e3 : 1;
103                  return value * multiplier;
104              }
105              return null;
106          })
107          .filter((bitrate): bitrate is number => bitrate !== null); // Remove any null values
108  
109      return { unit: "bit/s", results:  bitrates}
110  }
111  
112  interface ArgsRunBenchmarkAcrossVersions {
113      name: string,
114      clientPublicIP: string;
115      serverPublicIP: string;
116      uploadBytes: number,
117      downloadBytes: number,
118      unit: "bit/s" | "s",
119      iterations: number,
120      durationSecondsPerIteration: number,
121  }
122  
123  function runBenchmarkAcrossVersions(args: ArgsRunBenchmarkAcrossVersions): Benchmark {
124      console.error(`= Benchmark ${args.name}`)
125  
126      const results: Result[] = [];
127  
128      for (const version of versions) {
129          console.error(`== Version ${version.implementation}/${version.id}`)
130  
131          console.error(`=== Starting server ${version.implementation}/${version.id}`);
132  
133          const killCMD = `ssh -o StrictHostKeyChecking=no ec2-user@${args.serverPublicIP} 'kill $(cat pidfile); rm pidfile; rm server.log || true'`;
134          const killSTDOUT = execCommand(killCMD);
135          console.error(killSTDOUT);
136  
137          const serverCMD = `ssh -o StrictHostKeyChecking=no ec2-user@${args.serverPublicIP} 'nohup ./impl/${version.implementation}/${version.id}/perf --run-server --server-address 0.0.0.0:4001 > server.log 2>&1 & echo \$! > pidfile '`;
138          const serverSTDOUT = execCommand(serverCMD);
139          console.error(serverSTDOUT);
140  
141          for (const transportStack of version.transportStacks) {
142              const result = runClient({
143                  clientPublicIP: args.clientPublicIP,
144                  serverPublicIP: args.serverPublicIP,
145                  id: version.id,
146                  implementation: version.implementation,
147                  transportStack: transportStack,
148                  uploadBytes: args.uploadBytes,
149                  downloadBytes: args.downloadBytes,
150                  iterations: args.iterations,
151                  durationSecondsPerIteration: args.durationSecondsPerIteration,
152              });
153  
154              results.push({
155                  result,
156                  implementation: version.implementation,
157                  version: version.id,
158                  transportStack: transportStack,
159              });
160          }
161      };
162  
163      return {
164          name: args.name,
165          unit: args.unit,
166          results,
167          parameters: {
168              uploadBytes: args.uploadBytes,
169              downloadBytes: args.downloadBytes,
170          }
171      };
172  }
173  
174  interface ArgsRunBenchmark {
175      clientPublicIP: string;
176      serverPublicIP: string;
177      serverAddress?: string;
178      id: string,
179      implementation: string,
180      transportStack: string,
181      uploadBytes: number,
182      downloadBytes: number,
183      iterations: number,
184      durationSecondsPerIteration: number,
185  }
186  
187  function runClient(args: ArgsRunBenchmark): ResultValue[] {
188      console.error(`=== Starting client ${args.implementation}/${args.id}/${args.transportStack}`);
189  
190      const cmd = `./impl/${args.implementation}/${args.id}/perf --server-address ${args.serverPublicIP}:4001 --transport ${args.transportStack} --upload-bytes ${args.uploadBytes} --download-bytes ${args.downloadBytes}`
191      // Note 124 is timeout's exit code when timeout is hit which is not a failure here.
192      const withTimeout = `timeout ${args.durationSecondsPerIteration}s ${cmd} || [ $? -eq 124 ]`
193      const withForLoop = `for i in {1..${args.iterations}}; do ${withTimeout}; done`
194      const withSSH = `ssh -o StrictHostKeyChecking=no ec2-user@${args.clientPublicIP} '${withForLoop}'`
195  
196      const stdout = execCommand(withSSH);
197  
198      const lines = stdout.toString().trim().split('\n');
199  
200      const combined: ResultValue[]= [];
201  
202      for (const line of lines) {
203          const result = JSON.parse(line) as ResultValue;
204          combined.push(result);
205      }
206  
207      return combined;
208  }
209  
210  function execCommand(cmd: string): string {
211      try {
212          const stdout = execSync(cmd, {
213              encoding: 'utf8',
214              stdio: [process.stdin, 'pipe', process.stderr],
215          });
216          return stdout;
217      } catch (error) {
218          console.error((error as Error).message);
219          process.exit(1);
220      }
221  }
222  
223  function copyAndBuildPerfImplementations(ip: string) {
224      console.error(`= Building implementations on ${ip}`);
225  
226      const stdout = execCommand(`rsync -avz --progress --filter=':- .gitignore' -e "ssh -o StrictHostKeyChecking=no" ../impl ec2-user@${ip}:/home/ec2-user`);
227      console.error(stdout.toString());
228  
229      const stdout2 = execCommand(`ssh -o StrictHostKeyChecking=no ec2-user@${ip} 'cd impl && make'`);
230      console.error(stdout2.toString());
231  }
232  
233  const argv = yargs
234      .options({
235          'client-public-ip': {
236              type: 'string',
237              demandOption: true,
238              description: 'Client public IP address',
239          },
240          'server-public-ip': {
241              type: 'string',
242              demandOption: true,
243              description: 'Server public IP address',
244          },
245          'testing': {
246              type: 'boolean',
247              default: false,
248              description: 'Run in testing mode',
249              demandOption: false,
250          }
251      })
252      .command('help', 'Print usage information', yargs.help)
253      .parseSync();
254  
255  main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['testing'] as boolean);