1  using BlockchainUtils;
  2  using CodexClient;
  3  using CodexContractsPlugin;
  4  using CodexDiscordBotPlugin;
  5  using CodexPlugin;
  6  using Core;
  7  using GethPlugin;
  8  using KubernetesWorkflow.Types;
  9  using Logging;
 10  using MetricsPlugin;
 11  using WebUtils;
 12  
 13  namespace CodexNetDeployer
 14  {
 15      public class Deployer
 16      {
 17          private readonly Configuration config;
 18          private readonly PeerConnectivityChecker peerConnectivityChecker;
 19          private readonly EntryPoint entryPoint;
 20          private readonly LocalCodexBuilder localCodexBuilder;
 21  
 22          public Deployer(Configuration config)
 23          {
 24              this.config = config;
 25              peerConnectivityChecker = new PeerConnectivityChecker();
 26              localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath);
 27  
 28              ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
 29              ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
 30              ProjectPlugin.Load<GethPlugin.GethPlugin>();
 31              ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
 32              ProjectPlugin.Load<CodexDiscordBotPlugin.CodexDiscordBotPlugin>();
 33              entryPoint = CreateEntryPoint(new NullLog());
 34          }
 35  
 36          public void AnnouncePlugins()
 37          {
 38              var ep = CreateEntryPoint(new ConsoleLog());
 39  
 40              localCodexBuilder.Intialize();
 41  
 42              Log("Using plugins:" + Environment.NewLine);
 43              var metadata = ep.GetPluginMetadata();
 44              var longestKey = metadata.Keys.Max(k => k.Length);
 45              foreach (var entry in metadata)
 46              {
 47                  Console.Write(entry.Key);
 48                  Console.CursorLeft = longestKey + 5;
 49                  Console.WriteLine($"= {entry.Value}");
 50              }
 51  
 52              Log("");
 53          }
 54  
 55          public CodexDeployment Deploy()
 56          {
 57              localCodexBuilder.Build();
 58  
 59              Log("Initializing...");
 60              var startUtc = DateTime.UtcNow;
 61              var ci = entryPoint.CreateInterface();
 62  
 63              Log("Deploying Geth instance...");
 64              var gethDeployment = DeployGeth(ci);
 65              var gethNode = ci.WrapGethDeployment(gethDeployment, new BlockCache());
 66  
 67              var bootNode = ci.StartCodexNode();
 68              var versionInfo = bootNode.GetDebugInfo().Version;
 69              bootNode.Stop(waitTillStopped: true);
 70  
 71              Log("Geth started. Deploying Codex contracts...");
 72              var contractsDeployment = ci.DeployCodexContracts(gethNode, versionInfo);
 73              var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
 74              Log("Codex contracts deployed.");
 75  
 76              Log("Starting Codex nodes...");
 77              var codexStarter = new CodexNodeStarter(config, ci, gethNode, contracts, config.NumberOfValidators!.Value);
 78              var startResults = new List<CodexNodeStartResult>();
 79              for (var i = 0; i < config.NumberOfCodexNodes; i++)
 80              {
 81                  var result = codexStarter.Start(i);
 82                  if (result != null) startResults.Add(result);
 83              }
 84  
 85              Log("Codex nodes started.");
 86              var metricsService = StartMetricsService(ci, startResults);
 87  
 88              CheckPeerConnectivity(startResults);
 89              CheckContainerRestarts(startResults);
 90  
 91              var codexInstances = CreateCodexInstances(startResults);
 92  
 93              var discordBotContainer = DeployDiscordBot(ci, gethDeployment, contractsDeployment);
 94  
 95              return new CodexDeployment(codexInstances, gethDeployment, contractsDeployment, metricsService,
 96                  discordBotContainer, CreateMetadata(startUtc), config.DeployId);
 97          }
 98  
 99          private EntryPoint CreateEntryPoint(ILog log)
100          {
101              var kubeConfig = GetKubeConfig(config.KubeConfigFile);
102  
103              var configuration = new KubernetesWorkflow.Configuration(
104                  kubeConfig,
105                  operationTimeout: TimeSpan.FromMinutes(10),
106                  retryDelay: TimeSpan.FromSeconds(10),
107                  kubernetesNamespace: config.KubeNamespace);
108  
109              var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet(), new DefaultK8sTimeSet());
110              configuration.Hooks = new K8sHook(config.TestsTypePodLabel, config.DeployId, result.GetPluginMetadata());
111  
112              return result;
113          }
114  
115          private GethDeployment DeployGeth(CoreInterface ci)
116          {
117              return ci.DeployGeth(s =>
118              {
119                  s.IsMiner();
120                  s.WithName("geth");
121  
122                  if (config.IsPublicTestNet)
123                  {
124                      s.AsPublicTestNet(new GethTestNetConfig(
125                          discoveryPort: config.PublicGethDiscPort,
126                          listenPort: config.PublicGethListenPort
127                      ));
128                  }
129              });
130          }
131  
132          private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
133              CodexContractsDeployment contractsDeployment)
134          {
135              if (!config.DeployDiscordBot) return null;
136              Log("Deploying Discord bot...");
137  
138              var addr = gethDeployment.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag);
139              var info = new DiscordBotGethInfo(
140                  host: addr.Host,
141                  port: addr.Port,
142                  privKey: gethDeployment.Account.PrivateKey,
143                  marketplaceAddress: contractsDeployment.MarketplaceAddress,
144                  tokenAddress: contractsDeployment.TokenAddress,
145                  abi: contractsDeployment.Abi
146              );
147  
148              var rc = ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
149                  name: "discordbot-" + config.DeploymentName,
150                  token: config.DiscordBotToken,
151                  serverName: config.DiscordBotServerName,
152                  adminRoleName: config.DiscordBotAdminRoleName,
153                  adminChannelName: config.DiscordBotAdminChannelName,
154                  kubeNamespace: config.KubeNamespace,
155                  gethInfo: info,
156                  rewardChannelName: config.DiscordBotRewardChannelName)
157              {
158                  DataPath = config.DiscordBotDataPath
159              });
160  
161              Log("Discord bot deployed.");
162              return rc;
163          }
164  
165          private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
166          {
167              if (!config.MetricsScraper || !startResults.Any()) return null;
168  
169              Log("Starting metrics service...");
170  
171              var runningContainer = ci.DeployMetricsCollector(scrapeInterval: TimeSpan.FromSeconds(10.0), startResults.Select(r => r.CodexNode).ToArray());
172  
173              Log("Metrics service started.");
174  
175              return runningContainer;
176          }
177  
178          private CodexInstance[] CreateCodexInstances(List<CodexNodeStartResult> startResults)
179          {
180              // When freshly started, the Codex nodes are announcing themselves by an incorrect IP address.
181              // Only after fully initialized do they update to the provided NAT address.
182              // Therefore, we wait:
183              Thread.Sleep(TimeSpan.FromSeconds(5));
184  
185              return startResults.Select(r => CreateCodexInstance(r.CodexNode)).ToArray();
186          }
187  
188          private CodexInstance CreateCodexInstance(ICodexNode node)
189          {
190              //return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
191              throw new NotImplementedException();
192          }
193  
194          private string? GetKubeConfig(string kubeConfigFile)
195          {
196              if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
197              return kubeConfigFile;
198          }
199  
200          private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
201          {
202              if (!config.CheckPeerConnection || !codexContainers.Any()) return;
203  
204              Log("Starting peer connectivity check for deployed nodes...");
205              peerConnectivityChecker.CheckConnectivity(codexContainers);
206              Log("Check passed.");
207          }
208  
209          private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
210          {
211              var crashes = new List<ICodexNode>();
212              Log("Starting container crash check...");
213              foreach (var startResult in startResults)
214              {
215                  var hasCrashed = startResult.CodexNode.HasCrashed();
216                  if (hasCrashed) crashes.Add(startResult.CodexNode);
217              }
218  
219              if (!crashes.Any())
220              {
221                  Log("Check passed.");
222              }
223              else
224              {
225                  Log($"Check failed. The following containers have crashed: {crashes.Names()}");
226                  throw new Exception("Deployment failed: One or more containers crashed.");
227              }
228          }
229  
230          private DeploymentMetadata CreateMetadata(DateTime startUtc)
231          {
232              return new DeploymentMetadata(
233                  name: config.DeploymentName,
234                  startUtc: startUtc,
235                  finishedUtc: DateTime.UtcNow,
236                  kubeNamespace: config.KubeNamespace,
237                  numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
238                  numberOfValidators: config.NumberOfValidators!.Value,
239                  storageQuotaMB: config.StorageQuota!.Value,
240                  codexLogLevel: config.CodexLogLevel,
241                  initialTestTokens: config.InitialTestTokens,
242                  minPrice: config.MinPricePerBytePerSecond,
243                  maxCollateral: config.MaxCollateral,
244                  maxDuration: config.MaxDuration,
245                  blockTTL: config.BlockTTL,
246                  blockMI: config.BlockMI,
247                  blockMN: config.BlockMN);
248          }
249  
250          private void Log(string msg)
251          {
252              Console.WriteLine(msg);
253          }
254      }
255  
256      public class FastHttpTimeSet : IWebCallTimeSet
257      {
258          public TimeSpan HttpCallRetryDelay()
259          {
260              return TimeSpan.FromSeconds(2);
261          }
262  
263          public TimeSpan HttpRetryTimeout()
264          {
265              return TimeSpan.FromSeconds(30);
266          }
267  
268          public TimeSpan HttpCallTimeout()
269          {
270              return TimeSpan.FromSeconds(10);
271          }
272      }
273  }