1  using GethPlugin;
  2  using KubernetesWorkflow;
  3  using KubernetesWorkflow.Recipe;
  4  using Utils;
  5  
  6  namespace CodexPlugin
  7  {
  8      public class CodexContainerRecipe : ContainerRecipeFactory
  9      {
 10          public const string ApiPortTag = "codex_api_port";
 11          public const string ListenPortTag = "codex_listen_port";
 12          public const string MetricsPortTag = "codex_metrics_port";
 13          public const string DiscoveryPortTag = "codex_discovery_port";
 14  
 15          // Used by tests for time-constraint assertions.
 16          public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 17          public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 18          private readonly CodexDockerImage codexDockerImage;
 19  
 20          public override string AppName => "codex";
 21          public override string Image => codexDockerImage.GetCodexDockerImage();
 22  
 23          public CodexContainerRecipe(CodexDockerImage codexDockerImage)
 24          {
 25              this.codexDockerImage = codexDockerImage;
 26          }
 27  
 28          protected override void Initialize(StartupConfig startupConfig)
 29          {
 30              SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
 31              //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
 32  
 33              SetSchedulingAffinity(notIn: "false");
 34              SetSystemCriticalPriority();
 35  
 36              var config = startupConfig.Get<CodexStartupConfig>();
 37  
 38              var apiPort = CreateApiPort(config, ApiPortTag);
 39              AddEnvVar("CODEX_API_PORT", apiPort);
 40              AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
 41  
 42              var dataDir = $"datadir{ContainerNumber}";
 43              AddEnvVar("CODEX_DATA_DIR", dataDir);
 44              AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
 45  
 46              var discPort = CreateDiscoveryPort(config);
 47              AddEnvVar("CODEX_DISC_PORT", discPort);
 48              AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 49  
 50              if (config.PublicTestNet != null)
 51              {
 52                  // This makes the node announce itself to its public IP address.
 53                  AddEnvVar("NAT_IP_AUTO", "false");
 54                  AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
 55              }
 56              else
 57              {
 58                  // This makes the node announce itself to its local (pod) IP address.
 59                  AddEnvVar("NAT_IP_AUTO", "true");
 60              }
 61  
 62              var listenPort = CreateListenPort(config);
 63              AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 64  
 65              if (!string.IsNullOrEmpty(config.BootstrapSpr))
 66              {
 67                  AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
 68              }
 69              if (config.StorageQuota != null)
 70              {
 71                  AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
 72              }
 73              if (config.BlockTTL != null)
 74              {
 75                  AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
 76              }
 77              if (config.BlockMaintenanceInterval != null)
 78              {
 79                  AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
 80              }
 81              if (config.BlockMaintenanceNumber != null)
 82              {
 83                  AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
 84              }
 85              if (config.MetricsEnabled)
 86              {
 87                  var metricsPort = CreateApiPort(config, MetricsPortTag);
 88                  AddEnvVar("CODEX_METRICS", "true");
 89                  AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
 90                  AddEnvVar("CODEX_METRICS_PORT", metricsPort);
 91                  AddPodAnnotation("prometheus.io/scrape", "true");
 92                  AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
 93              }
 94  
 95              if (config.SimulateProofFailures != null)
 96              {
 97                  AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
 98              }
 99  
100              if (config.MarketplaceConfig != null)
101              {
102                  var mconfig = config.MarketplaceConfig;
103                  var gethStart = mconfig.GethNode.StartResult;
104                  var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
105                  var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
106  
107                  AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
108                  AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
109  
110                  var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
111  
112                  // Custom scripting in the Codex test image will write this variable to a private-key file,
113                  // and pass the correct filename to Codex.
114                  var account = marketplaceSetup.EthAccountSetup.GetNew();
115                  AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
116                  Additional(account);
117  
118                  SetCommandOverride(marketplaceSetup);
119                  if (marketplaceSetup.IsValidator)
120                  {
121                     AddEnvVar("CODEX_VALIDATOR", "true");
122                  }
123              }
124  
125              if (!string.IsNullOrEmpty(config.NameOverride))
126              {
127                  AddEnvVar("CODEX_NODENAME", config.NameOverride);
128              }
129          }
130  
131          private void SetCommandOverride(MarketplaceSetup ms)
132          {
133              if (ms.IsStorageNode)
134              {
135                  OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence", "prover");
136              }
137              else
138              {
139                  OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence");
140              }
141          }
142  
143          private Port CreateApiPort(CodexStartupConfig config, string tag)
144          {
145              if (config.PublicTestNet == null) return AddExposedPort(tag);
146              return AddInternalPort(tag);
147          }
148  
149          private Port CreateListenPort(CodexStartupConfig config)
150          {
151              if (config.PublicTestNet == null) return AddInternalPort(ListenPortTag);
152  
153              return AddExposedPort(config.PublicTestNet.PublicListenPort, ListenPortTag);
154          }
155  
156          private Port CreateDiscoveryPort(CodexStartupConfig config)
157          {
158              if (config.PublicTestNet == null) return AddInternalPort(DiscoveryPortTag, PortProtocol.UDP);
159  
160              return AddExposedPort(config.PublicTestNet.PublicDiscoveryPort, DiscoveryPortTag, PortProtocol.UDP);
161          }
162  
163          private ByteSize GetVolumeCapacity(CodexStartupConfig config)
164          {
165              if (config.StorageQuota != null) return config.StorageQuota.Multiply(1.2);
166              // Default Codex quota: 8 Gb, using +20% to be safe.
167              return 8.GB().Multiply(1.2);
168          }
169      }
170  }