spiffsgen.py
1 #!/usr/bin/env python 2 # 3 # spiffsgen is a tool used to generate a spiffs image from a directory 4 # 5 # Copyright 2019 Espressif Systems (Shanghai) PTE LTD 6 # 7 # Licensed under the Apache License, Version 2.0 (the "License"); 8 # you may not use this file except in compliance with the License. 9 # You may obtain a copy of the License at 10 # 11 # http:#www.apache.org/licenses/LICENSE-2.0 12 # 13 # Unless required by applicable law or agreed to in writing, software 14 # distributed under the License is distributed on an "AS IS" BASIS, 15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 # See the License for the specific language governing permissions and 17 # limitations under the License. 18 19 from __future__ import division 20 import os 21 import io 22 import math 23 import struct 24 import argparse 25 import ctypes 26 27 SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 28 SPIFFS_PH_FLAG_USED_FINAL = 0xFC 29 30 SPIFFS_PH_FLAG_LEN = 1 31 SPIFFS_PH_IX_SIZE_LEN = 4 32 SPIFFS_PH_IX_OBJ_TYPE_LEN = 1 33 SPIFFS_TYPE_FILE = 1 34 35 # Based on typedefs under spiffs_config.h 36 SPIFFS_OBJ_ID_LEN = 2 # spiffs_obj_id 37 SPIFFS_SPAN_IX_LEN = 2 # spiffs_span_ix 38 SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix 39 SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix 40 41 42 class SpiffsBuildConfig(): 43 def __init__(self, page_size, page_ix_len, block_size, 44 block_ix_len, meta_len, obj_name_len, obj_id_len, 45 span_ix_len, packed, aligned, endianness, use_magic, use_magic_len): 46 if block_size % page_size != 0: 47 raise RuntimeError("block size should be a multiple of page size") 48 49 self.page_size = page_size 50 self.block_size = block_size 51 self.obj_id_len = obj_id_len 52 self.span_ix_len = span_ix_len 53 self.packed = packed 54 self.aligned = aligned 55 self.obj_name_len = obj_name_len 56 self.meta_len = meta_len 57 self.page_ix_len = page_ix_len 58 self.block_ix_len = block_ix_len 59 self.endianness = endianness 60 self.use_magic = use_magic 61 self.use_magic_len = use_magic_len 62 63 self.PAGES_PER_BLOCK = self.block_size // self.page_size 64 self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) 65 self.OBJ_USABLE_PAGES_PER_BLOCK = self.PAGES_PER_BLOCK - self.OBJ_LU_PAGES_PER_BLOCK 66 67 self.OBJ_LU_PAGES_OBJ_IDS_LIM = self.page_size // self.obj_id_len 68 69 self.OBJ_DATA_PAGE_HEADER_LEN = self.obj_id_len + self.span_ix_len + SPIFFS_PH_FLAG_LEN 70 71 pad = 4 - (4 if self.OBJ_DATA_PAGE_HEADER_LEN % 4 == 0 else self.OBJ_DATA_PAGE_HEADER_LEN % 4) 72 73 self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED = self.OBJ_DATA_PAGE_HEADER_LEN + pad 74 self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD = pad 75 self.OBJ_DATA_PAGE_CONTENT_LEN = self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN 76 77 self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + 78 SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) 79 self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len 80 self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len 81 82 83 class SpiffsFullError(RuntimeError): 84 def __init__(self, message=None): 85 super(SpiffsFullError, self).__init__(message) 86 87 88 class SpiffsPage(): 89 _endianness_dict = { 90 "little": "<", 91 "big": ">" 92 } 93 94 _len_dict = { 95 1: "B", 96 2: "H", 97 4: "I", 98 8: "Q" 99 } 100 101 _type_dict = { 102 1: ctypes.c_ubyte, 103 2: ctypes.c_ushort, 104 4: ctypes.c_uint, 105 8: ctypes.c_ulonglong 106 } 107 108 def __init__(self, bix, build_config): 109 self.build_config = build_config 110 self.bix = bix 111 112 113 class SpiffsObjLuPage(SpiffsPage): 114 def __init__(self, bix, build_config): 115 SpiffsPage.__init__(self, bix, build_config) 116 117 self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM 118 self.obj_ids = list() 119 120 def _calc_magic(self, blocks_lim): 121 # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in 122 # spiffs_nucleus.h 123 magic = 0x20140529 ^ self.build_config.page_size 124 if self.build_config.use_magic_len: 125 magic = magic ^ (blocks_lim - self.bix) 126 magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic) 127 return magic.value 128 129 def register_page(self, page): 130 if not self.obj_ids_limit > 0: 131 raise SpiffsFullError() 132 133 obj_id = (page.obj_id, page.__class__) 134 self.obj_ids.append(obj_id) 135 self.obj_ids_limit -= 1 136 137 def to_binary(self): 138 global test 139 img = b"" 140 141 for (obj_id, page_type) in self.obj_ids: 142 if page_type == SpiffsObjIndexPage: 143 obj_id ^= (1 << ((self.build_config.obj_id_len * 8) - 1)) 144 img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + 145 SpiffsPage._len_dict[self.build_config.obj_id_len], obj_id) 146 147 assert(len(img) <= self.build_config.page_size) 148 149 img += b"\xFF" * (self.build_config.page_size - len(img)) 150 151 return img 152 153 def magicfy(self, blocks_lim): 154 # Only use magic value if no valid obj id has been written to the spot, which is the 155 # spot taken up by the last obj id on last lookup page. The parent is responsible 156 # for determining which is the last lookup page and calling this function. 157 remaining = self.obj_ids_limit 158 empty_obj_id_dict = { 159 1: 0xFF, 160 2: 0xFFFF, 161 4: 0xFFFFFFFF, 162 8: 0xFFFFFFFFFFFFFFFF 163 } 164 if (remaining >= 2): 165 for i in range(remaining): 166 if i == remaining - 2: 167 self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) 168 break 169 else: 170 self.obj_ids.append((empty_obj_id_dict[self.build_config.obj_id_len], SpiffsObjDataPage)) 171 self.obj_ids_limit -= 1 172 173 174 class SpiffsObjIndexPage(SpiffsPage): 175 def __init__(self, obj_id, span_ix, size, name, build_config): 176 SpiffsPage.__init__(self, 0, build_config) 177 self.obj_id = obj_id 178 self.span_ix = span_ix 179 self.name = name 180 self.size = size 181 182 if self.span_ix == 0: 183 self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM 184 else: 185 self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM 186 187 self.pages = list() 188 189 def register_page(self, page): 190 if not self.pages_lim > 0: 191 raise SpiffsFullError 192 193 self.pages.append(page.offset) 194 self.pages_lim -= 1 195 196 def to_binary(self): 197 obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) 198 img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + 199 SpiffsPage._len_dict[self.build_config.obj_id_len] + 200 SpiffsPage._len_dict[self.build_config.span_ix_len] + 201 SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], 202 obj_id, 203 self.span_ix, 204 SPIFFS_PH_FLAG_USED_FINAL_INDEX) 205 206 # Add padding before the object index page specific information 207 img += b"\xFF" * self.build_config.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD 208 209 # If this is the first object index page for the object, add filname, type 210 # and size information 211 if self.span_ix == 0: 212 img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + 213 SpiffsPage._len_dict[SPIFFS_PH_IX_SIZE_LEN] + 214 SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], 215 self.size, 216 SPIFFS_TYPE_FILE) 217 218 img += self.name.encode() + (b"\x00" * ((self.build_config.obj_name_len - len(self.name)) + self.build_config.meta_len)) 219 220 # Finally, add the page index of daa pages 221 for page in self.pages: 222 page = page >> int(math.log(self.build_config.page_size, 2)) 223 img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + 224 SpiffsPage._len_dict[self.build_config.page_ix_len], page) 225 226 assert(len(img) <= self.build_config.page_size) 227 228 img += b"\xFF" * (self.build_config.page_size - len(img)) 229 230 return img 231 232 233 class SpiffsObjDataPage(SpiffsPage): 234 def __init__(self, offset, obj_id, span_ix, contents, build_config): 235 SpiffsPage.__init__(self, 0, build_config) 236 self.obj_id = obj_id 237 self.span_ix = span_ix 238 self.contents = contents 239 self.offset = offset 240 241 def to_binary(self): 242 img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + 243 SpiffsPage._len_dict[self.build_config.obj_id_len] + 244 SpiffsPage._len_dict[self.build_config.span_ix_len] + 245 SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], 246 self.obj_id, 247 self.span_ix, 248 SPIFFS_PH_FLAG_USED_FINAL) 249 250 img += self.contents 251 252 assert(len(img) <= self.build_config.page_size) 253 254 img += b"\xFF" * (self.build_config.page_size - len(img)) 255 256 return img 257 258 259 class SpiffsBlock(): 260 def _reset(self): 261 self.cur_obj_index_span_ix = 0 262 self.cur_obj_data_span_ix = 0 263 self.cur_obj_id = 0 264 self.cur_obj_idx_page = None 265 266 def __init__(self, bix, blocks_lim, build_config): 267 self.build_config = build_config 268 self.offset = bix * self.build_config.block_size 269 self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK 270 self.pages = list() 271 self.bix = bix 272 273 lu_pages = list() 274 for i in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK): 275 page = SpiffsObjLuPage(self.bix, self.build_config) 276 lu_pages.append(page) 277 278 self.pages.extend(lu_pages) 279 280 self.lu_page_iter = iter(lu_pages) 281 self.lu_page = next(self.lu_page_iter) 282 283 self._reset() 284 285 def _register_page(self, page): 286 if isinstance(page, SpiffsObjDataPage): 287 self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError 288 289 try: 290 self.lu_page.register_page(page) 291 except SpiffsFullError: 292 self.lu_page = next(self.lu_page_iter) 293 try: 294 self.lu_page.register_page(page) 295 except AttributeError: # no next lookup page 296 # Since the amount of lookup pages is pre-computed at every block instance, 297 # this should never occur 298 raise RuntimeError("invalid attempt to add page to a block when there is no more space in lookup") 299 300 self.pages.append(page) 301 302 def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0): 303 if not self.remaining_pages > 0: 304 raise SpiffsFullError() 305 self._reset() 306 307 self.cur_obj_id = obj_id 308 self.cur_obj_index_span_ix = obj_index_span_ix 309 self.cur_obj_data_span_ix = obj_data_span_ix 310 311 page = SpiffsObjIndexPage(obj_id, self.cur_obj_index_span_ix, size, name, self.build_config) 312 self._register_page(page) 313 314 self.cur_obj_idx_page = page 315 316 self.remaining_pages -= 1 317 self.cur_obj_index_span_ix += 1 318 319 def update_obj(self, contents): 320 if not self.remaining_pages > 0: 321 raise SpiffsFullError() 322 page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), 323 self.cur_obj_id, self.cur_obj_data_span_ix, contents, self.build_config) 324 325 self._register_page(page) 326 327 self.cur_obj_data_span_ix += 1 328 self.remaining_pages -= 1 329 330 def end_obj(self): 331 self._reset() 332 333 def is_full(self): 334 return self.remaining_pages <= 0 335 336 def to_binary(self, blocks_lim): 337 img = b"" 338 339 if self.build_config.use_magic: 340 for (idx, page) in enumerate(self.pages): 341 if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: 342 page.magicfy(blocks_lim) 343 img += page.to_binary() 344 else: 345 for page in self.pages: 346 img += page.to_binary() 347 348 assert(len(img) <= self.build_config.block_size) 349 350 img += b"\xFF" * (self.build_config.block_size - len(img)) 351 return img 352 353 354 class SpiffsFS(): 355 def __init__(self, img_size, build_config): 356 if img_size % build_config.block_size != 0: 357 raise RuntimeError("image size should be a multiple of block size") 358 359 self.img_size = img_size 360 self.build_config = build_config 361 362 self.blocks = list() 363 self.blocks_lim = self.img_size // self.build_config.block_size 364 self.remaining_blocks = self.blocks_lim 365 self.cur_obj_id = 1 # starting object id 366 367 def _create_block(self): 368 if self.is_full(): 369 raise SpiffsFullError("the image size has been exceeded") 370 371 block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config) 372 self.blocks.append(block) 373 self.remaining_blocks -= 1 374 return block 375 376 def is_full(self): 377 return self.remaining_blocks <= 0 378 379 def create_file(self, img_path, file_path): 380 contents = None 381 382 if len(img_path) > self.build_config.obj_name_len: 383 raise RuntimeError("object name '%s' too long" % img_path) 384 385 name = img_path 386 387 with open(file_path, "rb") as obj: 388 contents = obj.read() 389 390 stream = io.BytesIO(contents) 391 392 try: 393 block = self.blocks[-1] 394 block.begin_obj(self.cur_obj_id, len(contents), name) 395 except (IndexError, SpiffsFullError): 396 block = self._create_block() 397 block.begin_obj(self.cur_obj_id, len(contents), name) 398 399 contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) 400 401 while contents_chunk: 402 try: 403 block = self.blocks[-1] 404 try: 405 # This can fail because either (1) all the pages in block have been 406 # used or (2) object index has been exhausted. 407 block.update_obj(contents_chunk) 408 except SpiffsFullError: 409 # If its (1), use the outer exception handler 410 if block.is_full(): 411 raise SpiffsFullError 412 # If its (2), write another object index page 413 block.begin_obj(self.cur_obj_id, len(contents), name, 414 obj_index_span_ix=block.cur_obj_index_span_ix, 415 obj_data_span_ix=block.cur_obj_data_span_ix) 416 continue 417 except (IndexError, SpiffsFullError): 418 # All pages in the block have been exhausted. Create a new block, copying 419 # the previous state of the block to a new one for the continuation of the 420 # current object 421 prev_block = block 422 block = self._create_block() 423 block.cur_obj_id = prev_block.cur_obj_id 424 block.cur_obj_idx_page = prev_block.cur_obj_idx_page 425 block.cur_obj_data_span_ix = prev_block.cur_obj_data_span_ix 426 block.cur_obj_index_span_ix = prev_block.cur_obj_index_span_ix 427 continue 428 429 contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) 430 431 block.end_obj() 432 433 self.cur_obj_id += 1 434 435 def to_binary(self): 436 img = b"" 437 for block in self.blocks: 438 img += block.to_binary(self.blocks_lim) 439 bix = len(self.blocks) 440 if self.build_config.use_magic: 441 # Create empty blocks with magic numbers 442 while self.remaining_blocks > 0: 443 block = SpiffsBlock(bix, self.blocks_lim, self.build_config) 444 img += block.to_binary(self.blocks_lim) 445 self.remaining_blocks -= 1 446 bix += 1 447 else: 448 # Just fill remaining spaces FF's 449 img += "\xFF" * (self.img_size - len(img)) 450 return img 451 452 453 def main(): 454 parser = argparse.ArgumentParser(description="SPIFFS Image Generator", 455 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 456 457 parser.add_argument("image_size", 458 help="Size of the created image") 459 460 parser.add_argument("base_dir", 461 help="Path to directory from which the image will be created") 462 463 parser.add_argument("output_file", 464 help="Created image output file path") 465 466 parser.add_argument("--page-size", 467 help="Logical page size. Set to value same as CONFIG_SPIFFS_PAGE_SIZE.", 468 type=int, 469 default=256) 470 471 parser.add_argument("--block-size", 472 help="Logical block size. Set to the same value as the flash chip's sector size (g_rom_flashchip.sector_size).", 473 type=int, 474 default=4096) 475 476 parser.add_argument("--obj-name-len", 477 help="File full path maximum length. Set to value same as CONFIG_SPIFFS_OBJ_NAME_LEN.", 478 type=int, 479 default=32) 480 481 parser.add_argument("--meta-len", 482 help="File metadata length. Set to value same as CONFIG_SPIFFS_META_LENGTH.", 483 type=int, 484 default=4) 485 486 parser.add_argument("--use-magic", 487 help="Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.", 488 action="store_true", 489 default=True) 490 491 parser.add_argument("--follow-symlinks", 492 help="Take into account symbolic links during partition image creation.", 493 action="store_true", 494 default=False) 495 496 parser.add_argument("--use-magic-len", 497 help="Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.", 498 action="store_true", 499 default=True) 500 501 parser.add_argument("--big-endian", 502 help="Specify if the target architecture is big-endian. If not specified, little-endian is assumed.", 503 action="store_true", 504 default=False) 505 506 args = parser.parse_args() 507 508 if not os.path.exists(args.base_dir): 509 raise RuntimeError("given base directory %s does not exist" % args.base_dir) 510 511 with open(args.output_file, "wb") as image_file: 512 image_size = int(args.image_size, 0) 513 spiffs_build_default = SpiffsBuildConfig(args.page_size, SPIFFS_PAGE_IX_LEN, 514 args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, 515 args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, 516 True, True, "big" if args.big_endian else "little", 517 args.use_magic, args.use_magic_len) 518 519 spiffs = SpiffsFS(image_size, spiffs_build_default) 520 521 for root, dirs, files in os.walk(args.base_dir, followlinks=args.follow_symlinks): 522 for f in files: 523 full_path = os.path.join(root, f) 524 spiffs.create_file("/" + os.path.relpath(full_path, args.base_dir).replace("\\", "/"), full_path) 525 526 image = spiffs.to_binary() 527 528 image_file.write(image) 529 530 531 if __name__ == "__main__": 532 main()