1  // Copyright 2023 Citra Emulator Project
  2  // Licensed under GPLv2 or any later version
  3  // Refer to the license.txt file included.
  4  
  5  #include <json.hpp>
  6  #include "common/file_util.h"
  7  #include "common/literals.h"
  8  #include "common/memory_detect.h"
  9  #include "common/microprofile.h"
 10  #include "common/settings.h"
 11  #include "common/string_util.h"
 12  #include "common/texture.h"
 13  #include "core/core.h"
 14  #include "core/frontend/image_interface.h"
 15  #include "core/hle/kernel/kernel.h"
 16  #include "core/hle/kernel/process.h"
 17  #include "video_core/custom_textures/custom_tex_manager.h"
 18  #include "video_core/rasterizer_cache/surface_params.h"
 19  #include "video_core/rasterizer_cache/utils.h"
 20  
 21  namespace VideoCore {
 22  
 23  namespace {
 24  
 25  MICROPROFILE_DEFINE(CustomTexManager_TickFrame, "CustomTexManager", "TickFrame",
 26                      MP_RGB(54, 16, 32));
 27  
 28  constexpr std::size_t MAX_UPLOADS_PER_TICK = 8;
 29  
 30  using namespace Common::Literals;
 31  
 32  bool IsPow2(u32 value) {
 33      return value != 0 && (value & (value - 1)) == 0;
 34  }
 35  
 36  CustomFileFormat MakeFileFormat(std::string_view ext) {
 37      if (ext == "png") {
 38          return CustomFileFormat::PNG;
 39      } else if (ext == "dds") {
 40          return CustomFileFormat::DDS;
 41      } else if (ext == "ktx") {
 42          return CustomFileFormat::KTX;
 43      }
 44      return CustomFileFormat::None;
 45  }
 46  
 47  MapType MakeMapType(std::string_view ext) {
 48      if (ext == "norm") {
 49          return MapType::Normal;
 50      }
 51      LOG_ERROR(Render, "Unknown material extension {}", ext);
 52      return MapType::Color;
 53  }
 54  
 55  } // Anonymous namespace
 56  
 57  CustomTexManager::CustomTexManager(Core::System& system_)
 58      : system{system_}, image_interface{*system.GetImageInterface()},
 59        async_custom_loading{Settings::values.async_custom_loading.GetValue()} {}
 60  
 61  CustomTexManager::~CustomTexManager() = default;
 62  
 63  void CustomTexManager::TickFrame() {
 64      MICROPROFILE_SCOPE(CustomTexManager_TickFrame);
 65      if (!textures_loaded) {
 66          return;
 67      }
 68      std::size_t num_uploads = 0;
 69      for (auto it = async_uploads.begin(); it != async_uploads.end();) {
 70          if (num_uploads >= MAX_UPLOADS_PER_TICK) {
 71              return;
 72          }
 73          switch (it->material->state) {
 74          case DecodeState::Decoded:
 75              it->func();
 76              num_uploads++;
 77              [[fallthrough]];
 78          case DecodeState::Failed:
 79              it = async_uploads.erase(it);
 80              continue;
 81          default:
 82              it++;
 83              break;
 84          }
 85      }
 86  }
 87  
 88  void CustomTexManager::FindCustomTextures() {
 89      if (textures_loaded) {
 90          return;
 91      }
 92      if (!workers) {
 93          CreateWorkers();
 94      }
 95  
 96      const u64 title_id = system.Kernel().GetCurrentProcess()->codeset->program_id;
 97      const auto textures = GetTextures(title_id);
 98      if (!ReadConfig(title_id)) {
 99          use_new_hash = false;
100          skip_mipmap = true;
101      }
102  
103      custom_textures.reserve(textures.size());
104      for (const FileUtil::FSTEntry& file : textures) {
105          if (file.isDirectory) {
106              continue;
107          }
108          custom_textures.push_back(std::make_unique<CustomTexture>(image_interface));
109          CustomTexture* const texture{custom_textures.back().get()};
110          if (!ParseFilename(file, texture)) {
111              continue;
112          }
113          for (const u64 hash : texture->hashes) {
114              auto& material = material_map[hash];
115              if (!material) {
116                  material = std::make_unique<Material>();
117              }
118              material->hash = hash;
119              material->AddMapTexture(texture);
120          }
121      }
122      textures_loaded = true;
123  }
124  
125  bool CustomTexManager::ParseFilename(const FileUtil::FSTEntry& file, CustomTexture* texture) {
126      auto parts = Common::SplitString(file.virtualName, '.');
127      if (parts.size() > 3) {
128          LOG_ERROR(Render, "Invalid filename {}, ignoring", file.virtualName);
129          return false;
130      }
131      // The last string should always be the file extension.
132      const CustomFileFormat file_format = MakeFileFormat(parts.back());
133      if (file_format == CustomFileFormat::None) {
134          return false;
135      }
136      if (file_format == CustomFileFormat::DDS && skip_mipmap) {
137          LOG_ERROR(Render, "Mipmap skip is incompatible with DDS textures, skipping!");
138          return false;
139      }
140      texture->file_format = file_format;
141      parts.pop_back();
142  
143      // This means the texture is a material type other than color.
144      texture->type = MapType::Color;
145      if (parts.size() > 1) {
146          texture->type = MakeMapType(parts.back());
147          parts.pop_back();
148      }
149  
150      // First look if this file is mapped to any number of hashes.
151      std::vector<u64>& hashes = texture->hashes;
152      const auto it = path_to_hash_map.find(file.virtualName);
153      if (it != path_to_hash_map.end()) {
154          hashes = it->second;
155      }
156  
157      // It's also possible for pack creators to retain the default texture name
158      // still map the texture to another hash. Support that as well.
159      u32 width;
160      u32 height;
161      u32 format;
162      unsigned long long hash{};
163      const bool is_parsed = std::sscanf(parts.back().c_str(), "tex1_%ux%u_%llX_%u", &width, &height,
164                                         &hash, &format) == 4;
165      const bool is_mapped =
166          !hashes.empty() && std::find(hashes.begin(), hashes.end(), hash) != hashes.end();
167      if (is_parsed && !is_mapped) {
168          hashes.push_back(hash);
169      }
170  
171      texture->path = file.physicalName;
172      return true;
173  }
174  
175  void CustomTexManager::PrepareDumping(u64 title_id) {
176      // If a pack exists in the load folder that uses the old hash, dump textures using the old hash.
177      // This occurs either if a configuration file doesn't exist or that file sets the old hash.
178      const std::string load_path =
179          fmt::format("{}textures/{:016X}/", GetUserPath(FileUtil::UserPath::LoadDir), title_id);
180      if (FileUtil::Exists(load_path) && !ReadConfig(title_id, true)) {
181          use_new_hash = false;
182      }
183  
184      // Write template config file
185      const std::string dump_path =
186          fmt::format("{}textures/{:016X}/", GetUserPath(FileUtil::UserPath::DumpDir), title_id);
187      const std::string pack_config = dump_path + "pack.json";
188      if (FileUtil::Exists(pack_config)) {
189          return;
190      }
191  
192      nlohmann::ordered_json json;
193      json["author"] = "citra";
194      json["version"] = "1.0.0";
195      json["description"] = "A graphics pack";
196  
197      auto& options = json["options"];
198      options["skip_mipmap"] = false;
199      options["flip_png_files"] = true;
200      options["use_new_hash"] = true;
201  
202      FileUtil::IOFile file{pack_config, "w"};
203      const std::string output = json.dump(4);
204      file.WriteString(output);
205  }
206  
207  void CustomTexManager::PreloadTextures(const std::atomic_bool& stop_run,
208                                         const VideoCore::DiskResourceLoadCallback& callback) {
209      u64 size_sum = 0;
210      std::size_t preloaded = 0;
211      const u64 sys_mem = Common::GetMemInfo().total_physical_memory;
212      const u64 recommended_min_mem = 2_GiB;
213  
214      // keep 2GiB memory for system stability if system RAM is 4GiB+ - use half of memory in other
215      // cases
216      const u64 max_mem =
217          (sys_mem / 2 < recommended_min_mem) ? (sys_mem / 2) : (sys_mem - recommended_min_mem);
218  
219      workers->QueueWork([&]() {
220          for (auto& [hash, material] : material_map) {
221              if (size_sum > max_mem) {
222                  LOG_WARNING(Render, "Aborting texture preload due to insufficient memory");
223                  return;
224              }
225              if (stop_run) {
226                  return;
227              }
228              material->LoadFromDisk(flip_png_files);
229              size_sum += material->size;
230              if (callback) {
231                  callback(VideoCore::LoadCallbackStage::Preload, preloaded, custom_textures.size());
232              }
233              preloaded++;
234          }
235      });
236      workers->WaitForRequests();
237      async_custom_loading = false;
238  }
239  
240  void CustomTexManager::DumpTexture(const SurfaceParams& params, u32 level, std::span<u8> data,
241                                     u64 data_hash) {
242      const u64 program_id = system.Kernel().GetCurrentProcess()->codeset->program_id;
243      const u32 data_size = static_cast<u32>(data.size());
244      const u32 width = params.width;
245      const u32 height = params.height;
246      const PixelFormat format = params.pixel_format;
247  
248      std::string dump_path = fmt::format(
249          "{}textures/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id);
250      if (!FileUtil::CreateFullPath(dump_path)) {
251          LOG_ERROR(Render, "Unable to create {}", dump_path);
252          return;
253      }
254  
255      dump_path +=
256          fmt::format("tex1_{}x{}_{:016X}_{}_mip{}.png", width, height, data_hash, format, level);
257      if (dumped_textures.contains(data_hash) || FileUtil::Exists(dump_path)) {
258          return;
259      }
260  
261      // Make sure the texture size is a power of 2.
262      // If not, the surface is probably a framebuffer
263      if (!IsPow2(width) || !IsPow2(height)) {
264          LOG_WARNING(Render, "Not dumping {:016X} because size isn't a power of 2 ({}x{})",
265                      data_hash, width, height);
266          return;
267      }
268  
269      const u32 decoded_size = width * height * 4;
270      std::vector<u8> pixels(data_size + decoded_size);
271      std::memcpy(pixels.data(), data.data(), data_size);
272  
273      auto dump = [this, width, height, params, data_size, decoded_size, pixels = std::move(pixels),
274                   dump_path = std::move(dump_path)]() mutable {
275          const std::span encoded = std::span{pixels}.first(data_size);
276          const std::span decoded = std::span{pixels}.last(decoded_size);
277          DecodeTexture(params, params.addr, params.end, encoded, decoded,
278                        params.type == SurfaceType::Color);
279          Common::FlipRGBA8Texture(decoded, width, height);
280          image_interface.EncodePNG(dump_path, width, height, decoded);
281      };
282      if (!workers) {
283          CreateWorkers();
284      }
285      workers->QueueWork(std::move(dump));
286      dumped_textures.insert(data_hash);
287  }
288  
289  Material* CustomTexManager::GetMaterial(u64 data_hash) {
290      const auto it = material_map.find(data_hash);
291      if (it == material_map.end()) {
292          LOG_WARNING(Render, "Unable to find replacement for surface with hash {:016X}", data_hash);
293          return nullptr;
294      }
295      return it->second.get();
296  }
297  
298  bool CustomTexManager::Decode(Material* material, std::function<bool()>&& upload) {
299      if (!async_custom_loading) {
300          material->LoadFromDisk(flip_png_files);
301          return upload();
302      }
303      if (material->IsUnloaded()) {
304          material->state = DecodeState::Pending;
305          workers->QueueWork([material, this] { material->LoadFromDisk(flip_png_files); });
306      }
307      async_uploads.push_back({
308          .material = material,
309          .func = std::move(upload),
310      });
311      return false;
312  }
313  
314  bool CustomTexManager::ReadConfig(u64 title_id, bool options_only) {
315      const std::string load_path =
316          fmt::format("{}textures/{:016X}/", GetUserPath(FileUtil::UserPath::LoadDir), title_id);
317      if (!FileUtil::Exists(load_path)) {
318          FileUtil::CreateFullPath(load_path);
319      }
320  
321      const std::string config_path = load_path + "pack.json";
322      FileUtil::IOFile config_file{config_path, "r"};
323      if (!config_file.IsOpen()) {
324          LOG_INFO(Render, "Unable to find pack config file, using legacy defaults");
325          return false;
326      }
327      std::string config(config_file.GetSize(), '\0');
328      const std::size_t read_size = config_file.ReadBytes(config.data(), config.size());
329      if (!read_size) {
330          return false;
331      }
332  
333      nlohmann::json json = nlohmann::json::parse(config, nullptr, false, true);
334  
335      const auto& options = json["options"];
336      skip_mipmap = options["skip_mipmap"].get<bool>();
337      flip_png_files = options["flip_png_files"].get<bool>();
338      use_new_hash = options["use_new_hash"].get<bool>();
339  
340      if (options_only) {
341          return true;
342      }
343  
344      const auto& textures = json["textures"];
345      for (const auto& material : textures.items()) {
346          std::size_t idx{};
347          const u64 hash = std::stoull(material.key(), &idx, 16);
348          if (!idx) {
349              LOG_ERROR(Render, "Key {} is invalid, skipping", material.key());
350              continue;
351          }
352          const auto parse = [&](const std::string& file) {
353              const std::string filename{FileUtil::GetFilename(file)};
354              auto [it, new_hash] = path_to_hash_map.try_emplace(filename);
355              it->second.push_back(hash);
356          };
357          const auto value = material.value();
358          if (value.is_string()) {
359              const auto file = value.get<std::string>();
360              parse(file);
361          } else if (value.is_array()) {
362              const auto files = value.get<std::vector<std::string>>();
363              for (const std::string& file : files) {
364                  parse(file);
365              }
366          } else {
367              LOG_ERROR(Render, "Material with key {} is invalid", material.key());
368          }
369      }
370      return true;
371  }
372  
373  std::vector<FileUtil::FSTEntry> CustomTexManager::GetTextures(u64 title_id) {
374      const std::string load_path =
375          fmt::format("{}textures/{:016X}/", GetUserPath(FileUtil::UserPath::LoadDir), title_id);
376      if (!FileUtil::Exists(load_path)) {
377          FileUtil::CreateFullPath(load_path);
378      }
379  
380      FileUtil::FSTEntry texture_dir;
381      std::vector<FileUtil::FSTEntry> textures;
382      FileUtil::ScanDirectoryTree(load_path, texture_dir, 64);
383      FileUtil::GetAllFilesFromNestedEntries(texture_dir, textures);
384      return textures;
385  }
386  
387  void CustomTexManager::CreateWorkers() {
388      const std::size_t num_workers = std::max(std::thread::hardware_concurrency(), 2U) - 1;
389      workers = std::make_unique<Common::ThreadWorker>(num_workers, "Custom textures");
390  }
391  
392  } // namespace VideoCore