1  const https = require('https');
 2  const fs = require('fs');
 3  const path = require('path');
 4  
 5  async function fetchFromGitHub(url, callback) {
 6      https.get(url, { headers: { 'User-Agent': 'Node.js' } }, (res) => {
 7          let data = '';
 8  
 9          res.on('data', (chunk) => {
10              data += chunk;
11          });
12  
13          res.on('end', () => {
14              callback(null, JSON.parse(data));
15          });
16      }).on('error', (err) => {
17          callback(err, null);
18      });
19  }
20  
21  async function fetchDirectoryContents(dirUrl, basePath, prefixToRemove) {
22      fetchFromGitHub(dirUrl, async (err, files) => {
23          if (err) {
24              console.error('Error fetching files:', err.message);
25              return;
26          }
27  
28          for (const file of files) {
29              const relativePath = file.path.replace(new RegExp(`^${prefixToRemove}`), '');
30              const filePath = path.join(basePath, relativePath);
31  
32              if (file.type === 'file') {
33                  await downloadAndSaveFile(file.download_url, filePath);
34              } else if (file.type === 'dir') {
35                  await fetchDirectoryContents(file.url, basePath, prefixToRemove); // Recursive call for subdirectories
36              }
37          }
38      });
39  }
40  
41  async function downloadAndSaveFile(url, filePath) {
42      const fullFilePath = path.join(__dirname, filePath);
43  
44      https.get(url, (res) => {
45          const directory = path.dirname(fullFilePath);
46  
47          // Ensure the directory exists
48          fs.mkdirSync(directory, { recursive: true });
49  
50          const fileStream = fs.createWriteStream(fullFilePath);
51          res.pipe(fileStream);
52  
53          fileStream.on('finish', () => {
54              fileStream.close();
55              console.log('Downloaded and saved:', filePath);
56          });
57      }).on('error', (err) => {
58          console.error('Error downloading file:', err.message);
59      });
60  }
61  
62  const repositories = [
63      {
64          baseUrl: 'https://api.github.com/repos/waku-org/nwaku/contents/docs/benchmarks',
65          baseSavePath: '/docs/learn/research/benchmarks/',
66          prefixToRemove: "docs/benchmarks/",
67          categoryFileContent: "{ \"label\": \"Benchmarks\", \"collapsed\": false }"
68      },
69      {
70          baseUrl: 'https://api.github.com/repos/waku-org/research/contents/docs',
71          baseSavePath: '/docs/learn/research/research-and-studies/',
72          prefixToRemove: "docs/",
73          categoryFileContent: "{ \"label\": \"Research and Studies\", \"collapsed\": false }"
74      }
75  ];
76  
77  repositories.forEach(repo => {
78      fs.rmSync(path.join(__dirname, repo.baseSavePath), { recursive: true, force: true });
79  });
80  
81  repositories.forEach(repo => {
82      fetchDirectoryContents(repo.baseUrl, repo.baseSavePath, repo.prefixToRemove, repo.categoryFileContent).then(() => {
83          const dir = path.join(__dirname, repo.baseSavePath);
84          fs.mkdirSync(dir, { recursive: true });
85          fs.writeFileSync(path.join(dir, "_category_.json"), repo.categoryFileContent);
86      });
87  });