1  #include "ZipFile.h"
  2  
  3  #include <HardwareSerial.h>
  4  #include <SDCardManager.h>
  5  #include <miniz.h>
  6  
  7  #include <algorithm>
  8  
  9  bool inflateOneShot(const uint8_t* inputBuf, const size_t deflatedSize, uint8_t* outputBuf, const size_t inflatedSize) {
 10    // Setup inflator
 11    const auto inflator = static_cast<tinfl_decompressor*>(malloc(sizeof(tinfl_decompressor)));
 12    if (!inflator) {
 13      Serial.printf("[%lu] [ZIP] Failed to allocate memory for inflator\n", millis());
 14      return false;
 15    }
 16    memset(inflator, 0, sizeof(tinfl_decompressor));
 17    tinfl_init(inflator);
 18  
 19    size_t inBytes = deflatedSize;
 20    size_t outBytes = inflatedSize;
 21    const tinfl_status status = tinfl_decompress(inflator, inputBuf, &inBytes, nullptr, outputBuf, &outBytes,
 22                                                 TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);
 23    free(inflator);
 24  
 25    if (status != TINFL_STATUS_DONE) {
 26      Serial.printf("[%lu] [ZIP] tinfl_decompress() failed with status %d\n", millis(), status);
 27      return false;
 28    }
 29  
 30    return true;
 31  }
 32  
 33  bool ZipFile::loadAllFileStatSlims() {
 34    const bool wasOpen = isOpen();
 35    if (!wasOpen && !open()) {
 36      return false;
 37    }
 38  
 39    if (!loadZipDetails()) {
 40      if (!wasOpen) {
 41        close();
 42      }
 43      return false;
 44    }
 45  
 46    file.seek(zipDetails.centralDirOffset);
 47  
 48    uint32_t sig;
 49    char itemName[256];
 50    fileStatSlimCache.clear();
 51    fileStatSlimCache.reserve(zipDetails.totalEntries);
 52  
 53    while (file.available()) {
 54      file.read(&sig, 4);
 55      if (sig != 0x02014b50) break;  // End of list
 56  
 57      FileStatSlim fileStat = {};
 58  
 59      file.seekCur(6);
 60      file.read(&fileStat.method, 2);
 61      file.seekCur(8);
 62      file.read(&fileStat.compressedSize, 4);
 63      file.read(&fileStat.uncompressedSize, 4);
 64      uint16_t nameLen, m, k;
 65      file.read(&nameLen, 2);
 66      file.read(&m, 2);
 67      file.read(&k, 2);
 68      file.seekCur(8);
 69      file.read(&fileStat.localHeaderOffset, 4);
 70  
 71      // Bounds check to prevent buffer overflow
 72      if (nameLen >= 255) {
 73        file.seekCur(nameLen + m + k);  // Skip this entry entirely
 74        continue;
 75      }
 76  
 77      file.read(itemName, nameLen);
 78      itemName[nameLen] = '\0';
 79  
 80      fileStatSlimCache.emplace(itemName, fileStat);
 81  
 82      // Skip the rest of this entry (extra field + comment)
 83      file.seekCur(m + k);
 84    }
 85  
 86    if (!wasOpen) {
 87      close();
 88    }
 89    return true;
 90  }
 91  
 92  bool ZipFile::loadFileStatSlim(const char* filename, FileStatSlim* fileStat) {
 93    if (!fileStatSlimCache.empty()) {
 94      const auto it = fileStatSlimCache.find(filename);
 95      if (it != fileStatSlimCache.end()) {
 96        *fileStat = it->second;
 97        return true;
 98      }
 99      return false;
100    }
101  
102    const bool wasOpen = isOpen();
103    if (!wasOpen && !open()) {
104      return false;
105    }
106  
107    if (!loadZipDetails()) {
108      if (!wasOpen) {
109        close();
110      }
111      return false;
112    }
113  
114    file.seek(zipDetails.centralDirOffset);
115  
116    uint32_t sig;
117    char itemName[256];
118    bool found = false;
119  
120    while (file.available()) {
121      file.read(&sig, 4);
122      if (sig != 0x02014b50) break;  // End of list
123  
124      file.seekCur(6);
125      file.read(&fileStat->method, 2);
126      file.seekCur(8);
127      file.read(&fileStat->compressedSize, 4);
128      file.read(&fileStat->uncompressedSize, 4);
129      uint16_t nameLen, m, k;
130      file.read(&nameLen, 2);
131      file.read(&m, 2);
132      file.read(&k, 2);
133      file.seekCur(8);
134      file.read(&fileStat->localHeaderOffset, 4);
135  
136      // Bounds check to prevent buffer overflow
137      if (nameLen >= 255) {
138        file.seekCur(nameLen + m + k);  // Skip this entry entirely
139        continue;
140      }
141  
142      file.read(itemName, nameLen);
143      itemName[nameLen] = '\0';
144  
145      if (strcmp(itemName, filename) == 0) {
146        found = true;
147        break;
148      }
149  
150      // Skip the rest of this entry (extra field + comment)
151      file.seekCur(m + k);
152    }
153  
154    if (!wasOpen) {
155      close();
156    }
157    return found;
158  }
159  
160  long ZipFile::getDataOffset(const FileStatSlim& fileStat) {
161    const bool wasOpen = isOpen();
162    if (!wasOpen && !open()) {
163      return -1;
164    }
165  
166    constexpr auto localHeaderSize = 30;
167  
168    uint8_t pLocalHeader[localHeaderSize];
169    const uint64_t fileOffset = fileStat.localHeaderOffset;
170  
171    file.seek(fileOffset);
172    const size_t read = file.read(pLocalHeader, localHeaderSize);
173    if (!wasOpen) {
174      close();
175    }
176  
177    if (read != localHeaderSize) {
178      Serial.printf("[%lu] [ZIP] Something went wrong reading the local header\n", millis());
179      return -1;
180    }
181  
182    if (pLocalHeader[0] + (pLocalHeader[1] << 8) + (pLocalHeader[2] << 16) + (pLocalHeader[3] << 24) !=
183        0x04034b50 /* MZ_ZIP_LOCAL_DIR_HEADER_SIG */) {
184      Serial.printf("[%lu] [ZIP] Not a valid zip file header\n", millis());
185      return -1;
186    }
187  
188    const uint16_t filenameLength = pLocalHeader[26] + (pLocalHeader[27] << 8);
189    const uint16_t extraOffset = pLocalHeader[28] + (pLocalHeader[29] << 8);
190    return fileOffset + localHeaderSize + filenameLength + extraOffset;
191  }
192  
193  bool ZipFile::loadZipDetails() {
194    if (zipDetails.isSet) {
195      return true;
196    }
197  
198    const bool wasOpen = isOpen();
199    if (!wasOpen && !open()) {
200      return false;
201    }
202  
203    const size_t fileSize = file.size();
204    if (fileSize < 22) {
205      Serial.printf("[%lu] [ZIP] File too small to be a valid zip\n", millis());
206      if (!wasOpen) {
207        close();
208      }
209      return false;  // Minimum EOCD size is 22 bytes
210    }
211  
212    // We scan the last 1KB (or the whole file if smaller) for the EOCD signature
213    // 0x06054b50 is stored as 0x50, 0x4b, 0x05, 0x06 in little-endian
214    const int scanRange = fileSize > 1024 ? 1024 : fileSize;
215    const auto buffer = static_cast<uint8_t*>(malloc(scanRange));
216    if (!buffer) {
217      Serial.printf("[%lu] [ZIP] Failed to allocate memory for EOCD scan buffer\n", millis());
218      if (!wasOpen) {
219        close();
220      }
221      return false;
222    }
223  
224    file.seek(fileSize - scanRange);
225    file.read(buffer, scanRange);
226  
227    // Scan backwards for the signature
228    int foundOffset = -1;
229    for (int i = scanRange - 22; i >= 0; i--) {
230      constexpr uint8_t signature[4] = {0x50, 0x4b, 0x05, 0x06};  // Little-endian EOCD signature
231      if (memcmp(&buffer[i], signature, 4) == 0) {
232        foundOffset = i;
233        break;
234      }
235    }
236  
237    if (foundOffset == -1) {
238      Serial.printf("[%lu] [ZIP] EOCD signature not found in zip file\n", millis());
239      free(buffer);
240      if (!wasOpen) {
241        close();
242      }
243      return false;
244    }
245  
246    // Now extract the values we need from the EOCD record
247    // Relative positions within EOCD:
248    // Offset 10: Total number of entries (2 bytes)
249    // Offset 16: Offset of start of central directory with respect to the starting disk number (4 bytes)
250    memcpy(&zipDetails.totalEntries, &buffer[foundOffset + 10], sizeof(zipDetails.totalEntries));
251    memcpy(&zipDetails.centralDirOffset, &buffer[foundOffset + 16], sizeof(zipDetails.centralDirOffset));
252    zipDetails.isSet = true;
253  
254    free(buffer);
255    if (!wasOpen) {
256      close();
257    }
258    return true;
259  }
260  
261  uint16_t ZipFile::getTotalEntries() {
262    if (!zipDetails.isSet) {
263      loadZipDetails();
264    }
265    return zipDetails.totalEntries;
266  }
267  
268  bool ZipFile::open() {
269    if (!SdMan.openFileForRead("ZIP", filePath, file)) {
270      return false;
271    }
272    return true;
273  }
274  
275  bool ZipFile::close() {
276    if (file) {
277      file.close();
278    }
279    return true;
280  }
281  
282  bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) {
283    FileStatSlim fileStat = {};
284    if (!loadFileStatSlim(filename, &fileStat)) {
285      return false;
286    }
287  
288    *size = static_cast<size_t>(fileStat.uncompressedSize);
289    return true;
290  }
291  
292  int ZipFile::fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes) {
293    if (targets.empty()) {
294      return 0;
295    }
296  
297    const bool wasOpen = isOpen();
298    if (!wasOpen && !open()) {
299      return 0;
300    }
301  
302    if (!loadZipDetails()) {
303      if (!wasOpen) {
304        close();
305      }
306      return 0;
307    }
308  
309    file.seek(zipDetails.centralDirOffset);
310  
311    uint32_t sig;
312    char itemName[256];
313    int matched = 0;
314  
315    while (file.available()) {
316      if (file.read(&sig, 4) != 4) break;
317      if (sig != 0x02014b50) break;  // End of central directory
318  
319      // Skip: version made by (2), version needed (2), flags (2), method (2), time (2), date (2), crc32 (4)
320      file.seekCur(16);
321      // Skip compressedSize (4), read uncompressedSize (4)
322      file.seekCur(4);
323      uint32_t uncompressedSize;
324      if (file.read(&uncompressedSize, 4) != 4) break;
325      uint16_t nameLen, m, k;
326      if (file.read(&nameLen, 2) != 2) break;
327      if (file.read(&m, 2) != 2) break;
328      if (file.read(&k, 2) != 2) break;
329      // Skip: comment len already read in k, disk# (2), internal attr (2), external attr (4), local header offset (4)
330      file.seekCur(12);
331  
332      // Bounds check to prevent buffer overflow
333      if (nameLen >= 255) {
334        file.seekCur(nameLen + m + k);  // Skip this entry entirely
335        continue;
336      }
337  
338      if (file.read(itemName, nameLen) != nameLen) break;
339      itemName[nameLen] = '\0';
340  
341      // Compute hash on-the-fly from filename
342      const uint64_t entryHash = fnvHash64(itemName, nameLen);
343  
344      // Binary search for matching target
345      SizeTarget key = {entryHash, nameLen, 0};
346      auto it = std::lower_bound(targets.begin(), targets.end(), key);
347  
348      // Check for match (hash and len must match)
349      if (it != targets.end() && it->hash == entryHash && it->len == nameLen) {
350        // Bounds check before write
351        if (it->index < sizes.size()) {
352          sizes[it->index] = uncompressedSize;
353          matched++;
354        }
355      }
356  
357      // Skip the rest of this entry (extra field + comment)
358      file.seekCur(m + k);
359    }
360  
361    if (!wasOpen) {
362      close();
363    }
364    return matched;
365  }
366  
367  int ZipFile::findFirstExisting(const char* const* paths, int pathCount) {
368    if (!paths || pathCount <= 0 || pathCount > 65535) {
369      return -1;
370    }
371  
372    const bool wasOpen = isOpen();
373    if (!wasOpen && !open()) {
374      return -1;
375    }
376  
377    if (!loadZipDetails()) {
378      if (!wasOpen) {
379        close();
380      }
381      return -1;
382    }
383  
384    // Build sorted vector of targets with hashes for binary search
385    std::vector<SizeTarget> targets;
386    for (int i = 0; i < pathCount; i++) {
387      const char* path = paths[i];
388      if (!path) continue;
389      const size_t len = strlen(path);
390      if (len > 255) continue;  // Skip paths that are too long for itemName[256]
391      targets.push_back({fnvHash64(path, len), static_cast<uint16_t>(len), static_cast<uint16_t>(i)});
392    }
393    std::sort(targets.begin(), targets.end());
394  
395    file.seek(zipDetails.centralDirOffset);
396  
397    uint32_t sig;
398    char itemName[256];
399    int foundIndex = -1;
400    int lowestPriority = pathCount;  // Lower index = higher priority
401  
402    while (file.available()) {
403      if (file.read(&sig, 4) != 4) break;
404      if (sig != 0x02014b50) break;  // End of central directory
405  
406      // Skip to name length (skip 24 bytes from after signature)
407      if (!file.seekCur(24)) break;
408      uint16_t nameLen, m, k;
409      if (file.read(&nameLen, 2) != 2) break;
410      if (file.read(&m, 2) != 2) break;
411      if (file.read(&k, 2) != 2) break;
412      // Skip remaining header (12 bytes)
413      if (!file.seekCur(12)) break;
414  
415      // Bounds check to prevent buffer overflow
416      if (nameLen > 255) {
417        file.seekCur(nameLen + m + k);
418        continue;
419      }
420  
421      if (file.read(itemName, nameLen) != nameLen) break;
422      itemName[nameLen] = '\0';
423  
424      // Compute hash on-the-fly from filename
425      const uint64_t entryHash = fnvHash64(itemName, nameLen);
426  
427      // Binary search for matching target
428      SizeTarget key = {entryHash, nameLen, 0};
429      auto it = std::lower_bound(targets.begin(), targets.end(), key);
430  
431      // Check for match (hash and len must match)
432      if (it != targets.end() && it->hash == entryHash && it->len == nameLen) {
433        // Verify string match (hash collision protection) with bounds check
434        if (it->index < pathCount && strcmp(itemName, paths[it->index]) == 0) {
435          // Keep track of lowest index (highest priority)
436          if (it->index < lowestPriority) {
437            lowestPriority = it->index;
438            foundIndex = it->index;
439            if (lowestPriority == 0) break;  // Can't find higher priority
440          }
441        }
442      }
443  
444      // Skip the rest of this entry (extra field + comment)
445      file.seekCur(m + k);
446    }
447  
448    if (!wasOpen) {
449      close();
450    }
451    return foundIndex;
452  }
453  
454  uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const bool trailingNullByte) {
455    const bool wasOpen = isOpen();
456    if (!wasOpen && !open()) {
457      return nullptr;
458    }
459  
460    FileStatSlim fileStat = {};
461    if (!loadFileStatSlim(filename, &fileStat)) {
462      if (!wasOpen) {
463        close();
464      }
465      return nullptr;
466    }
467  
468    const long fileOffset = getDataOffset(fileStat);
469    if (fileOffset < 0) {
470      if (!wasOpen) {
471        close();
472      }
473      return nullptr;
474    }
475  
476    file.seek(fileOffset);
477  
478    const auto deflatedDataSize = fileStat.compressedSize;
479    const auto inflatedDataSize = fileStat.uncompressedSize;
480    const auto dataSize = trailingNullByte ? inflatedDataSize + 1 : inflatedDataSize;
481    const auto data = static_cast<uint8_t*>(malloc(dataSize));
482    if (data == nullptr) {
483      Serial.printf("[%lu] [ZIP] Failed to allocate memory for output buffer (%zu bytes)\n", millis(), dataSize);
484      if (!wasOpen) {
485        close();
486      }
487      return nullptr;
488    }
489  
490    if (fileStat.method == 0) {  // MZ_NO_COMPRESSION = 0
491      // no deflation, just read content
492      const size_t dataRead = file.read(data, inflatedDataSize);
493      if (!wasOpen) {
494        close();
495      }
496  
497      if (dataRead != inflatedDataSize) {
498        Serial.printf("[%lu] [ZIP] Failed to read data\n", millis());
499        free(data);
500        return nullptr;
501      }
502  
503      // Continue out of block with data set
504    } else if (fileStat.method == MZ_DEFLATED) {
505      // Read out deflated content from file
506      const auto deflatedData = static_cast<uint8_t*>(malloc(deflatedDataSize));
507      if (deflatedData == nullptr) {
508        Serial.printf("[%lu] [ZIP] Failed to allocate memory for decompression buffer\n", millis());
509        if (!wasOpen) {
510          close();
511        }
512        return nullptr;
513      }
514  
515      const size_t dataRead = file.read(deflatedData, deflatedDataSize);
516      if (!wasOpen) {
517        close();
518      }
519  
520      if (dataRead != deflatedDataSize) {
521        Serial.printf("[%lu] [ZIP] Failed to read data, expected %d got %d\n", millis(), deflatedDataSize, dataRead);
522        free(deflatedData);
523        free(data);
524        return nullptr;
525      }
526  
527      const bool success = inflateOneShot(deflatedData, deflatedDataSize, data, inflatedDataSize);
528      free(deflatedData);
529  
530      if (!success) {
531        Serial.printf("[%lu] [ZIP] Failed to inflate file\n", millis());
532        free(data);
533        if (!wasOpen) close();
534        return nullptr;
535      }
536  
537      // Continue out of block with data set
538    } else {
539      Serial.printf("[%lu] [ZIP] Unsupported compression method\n", millis());
540      if (!wasOpen) {
541        close();
542      }
543      return nullptr;
544    }
545  
546    if (trailingNullByte) data[inflatedDataSize] = '\0';
547    if (size) *size = inflatedDataSize;
548    return data;
549  }
550  
551  bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t chunkSize) {
552    const bool wasOpen = isOpen();
553    if (!wasOpen && !open()) {
554      return false;
555    }
556  
557    FileStatSlim fileStat = {};
558    if (!loadFileStatSlim(filename, &fileStat)) {
559      if (!wasOpen) close();
560      return false;
561    }
562  
563    const long fileOffset = getDataOffset(fileStat);
564    if (fileOffset < 0) {
565      if (!wasOpen) close();
566      return false;
567    }
568  
569    file.seek(fileOffset);
570    const auto deflatedDataSize = fileStat.compressedSize;
571    const auto inflatedDataSize = fileStat.uncompressedSize;
572  
573    if (fileStat.method == 0) {  // MZ_NO_COMPRESSION = 0
574      // no deflation, just read content
575      const auto buffer = static_cast<uint8_t*>(malloc(chunkSize));
576      if (!buffer) {
577        Serial.printf("[%lu] [ZIP] Failed to allocate memory for buffer\n", millis());
578        if (!wasOpen) {
579          close();
580        }
581        return false;
582      }
583  
584      size_t remaining = inflatedDataSize;
585      while (remaining > 0) {
586        const size_t dataRead = file.read(buffer, remaining < chunkSize ? remaining : chunkSize);
587        if (dataRead == 0) {
588          Serial.printf("[%lu] [ZIP] Could not read more bytes\n", millis());
589          free(buffer);
590          if (!wasOpen) {
591            close();
592          }
593          return false;
594        }
595  
596        out.write(buffer, dataRead);
597        remaining -= dataRead;
598      }
599  
600      if (!wasOpen) {
601        close();
602      }
603      free(buffer);
604      return true;
605    }
606  
607    if (fileStat.method == MZ_DEFLATED) {
608      // Setup inflator
609      const auto inflator = static_cast<tinfl_decompressor*>(malloc(sizeof(tinfl_decompressor)));
610      if (!inflator) {
611        Serial.printf("[%lu] [ZIP] Failed to allocate memory for inflator\n", millis());
612        if (!wasOpen) {
613          close();
614        }
615        return false;
616      }
617      memset(inflator, 0, sizeof(tinfl_decompressor));
618      tinfl_init(inflator);
619  
620      // Setup file read buffer
621      const auto fileReadBuffer = static_cast<uint8_t*>(malloc(chunkSize));
622      if (!fileReadBuffer) {
623        Serial.printf("[%lu] [ZIP] Failed to allocate memory for zip file read buffer\n", millis());
624        free(inflator);
625        if (!wasOpen) {
626          close();
627        }
628        return false;
629      }
630  
631      const auto outputBuffer = static_cast<uint8_t*>(malloc(TINFL_LZ_DICT_SIZE));
632      if (!outputBuffer) {
633        Serial.printf("[%lu] [ZIP] Failed to allocate memory for dictionary\n", millis());
634        free(inflator);
635        free(fileReadBuffer);
636        if (!wasOpen) {
637          close();
638        }
639        return false;
640      }
641      memset(outputBuffer, 0, TINFL_LZ_DICT_SIZE);
642  
643      size_t fileRemainingBytes = deflatedDataSize;
644      size_t processedOutputBytes = 0;
645      size_t fileReadBufferFilledBytes = 0;
646      size_t fileReadBufferCursor = 0;
647      size_t outputCursor = 0;  // Current offset in the circular dictionary
648  
649      while (true) {
650        // Load more compressed bytes when needed
651        if (fileReadBufferCursor >= fileReadBufferFilledBytes) {
652          if (fileRemainingBytes == 0) {
653            // Should not be hit, but a safe protection
654            break;  // EOF
655          }
656  
657          fileReadBufferFilledBytes =
658              file.read(fileReadBuffer, fileRemainingBytes < chunkSize ? fileRemainingBytes : chunkSize);
659          fileRemainingBytes -= fileReadBufferFilledBytes;
660          fileReadBufferCursor = 0;
661  
662          if (fileReadBufferFilledBytes == 0) {
663            // Bad read
664            break;  // EOF
665          }
666        }
667  
668        // Available bytes in fileReadBuffer to process
669        size_t inBytes = fileReadBufferFilledBytes - fileReadBufferCursor;
670        // Space remaining in outputBuffer
671        size_t outBytes = TINFL_LZ_DICT_SIZE - outputCursor;
672  
673        const tinfl_status status = tinfl_decompress(inflator, fileReadBuffer + fileReadBufferCursor, &inBytes,
674                                                     outputBuffer, outputBuffer + outputCursor, &outBytes,
675                                                     fileRemainingBytes > 0 ? TINFL_FLAG_HAS_MORE_INPUT : 0);
676  
677        // Update input position
678        fileReadBufferCursor += inBytes;
679  
680        // Write output chunk
681        if (outBytes > 0) {
682          processedOutputBytes += outBytes;
683          if (out.write(outputBuffer + outputCursor, outBytes) != outBytes) {
684            Serial.printf("[%lu] [ZIP] Failed to write all output bytes to stream\n", millis());
685            if (!wasOpen) {
686              close();
687            }
688            free(outputBuffer);
689            free(fileReadBuffer);
690            free(inflator);
691            return false;
692          }
693          // Update output position in buffer (with wraparound)
694          outputCursor = (outputCursor + outBytes) & (TINFL_LZ_DICT_SIZE - 1);
695        }
696  
697        if (status < 0) {
698          Serial.printf("[%lu] [ZIP] tinfl_decompress() failed with status %d\n", millis(), status);
699          if (!wasOpen) {
700            close();
701          }
702          free(outputBuffer);
703          free(fileReadBuffer);
704          free(inflator);
705          return false;
706        }
707  
708        if (status == TINFL_STATUS_DONE) {
709          Serial.printf("[%lu] [ZIP] Decompressed %d bytes into %d bytes\n", millis(), deflatedDataSize,
710                        inflatedDataSize);
711          if (!wasOpen) {
712            close();
713          }
714          free(inflator);
715          free(fileReadBuffer);
716          free(outputBuffer);
717          return true;
718        }
719      }
720  
721      // If we get here, EOF reached without TINFL_STATUS_DONE
722      Serial.printf("[%lu] [ZIP] Unexpected EOF\n", millis());
723      if (!wasOpen) {
724        close();
725      }
726      free(outputBuffer);
727      free(fileReadBuffer);
728      free(inflator);
729      return false;
730    }
731  
732    if (!wasOpen) {
733      close();
734    }
735  
736    Serial.printf("[%lu] [ZIP] Unsupported compression method\n", millis());
737    return false;
738  }