1  // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
  2  //
  3  // Licensed under the Apache License, Version 2.0 (the "License");
  4  // you may not use this file except in compliance with the License.
  5  // You may obtain a copy of the License at
  6  
  7  //     http://www.apache.org/licenses/LICENSE-2.0
  8  //
  9  // Unless required by applicable law or agreed to in writing, software
 10  // distributed under the License is distributed on an "AS IS" BASIS,
 11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12  // See the License for the specific language governing permissions and
 13  // limitations under the License.
 14  #include <stdbool.h>
 15  #include <string.h>
 16  #include <assert.h>
 17  #include <stdio.h>
 18  #include <sys/param.h>
 19  #include "esp_attr.h"
 20  #include "esp_heap_caps.h"
 21  #include "multi_heap.h"
 22  #include "esp_log.h"
 23  #include "heap_private.h"
 24  #include "esp_system.h"
 25  
 26  /*
 27  This file, combined with a region allocator that supports multiple heaps, solves the problem that the ESP32 has RAM
 28  that's slightly heterogeneous. Some RAM can be byte-accessed, some allows only 32-bit accesses, some can execute memory,
 29  some can be remapped by the MMU to only be accessed by a certain PID etc. In order to allow the most flexible memory
 30  allocation possible, this code makes it possible to request memory that has certain capabilities. The code will then use
 31  its knowledge of how the memory is configured along with a priority scheme to allocate that memory in the most sane way
 32  possible. This should optimize the amount of RAM accessible to the code without hardwiring addresses.
 33  */
 34  
 35  static esp_alloc_failed_hook_t alloc_failed_callback;
 36  
 37  /*
 38    This takes a memory chunk in a region that can be addressed as both DRAM as well as IRAM. It will convert it to
 39    IRAM in such a way that it can be later freed. It assumes both the address as well as the length to be word-aligned.
 40    It returns a region that's 1 word smaller than the region given because it stores the original Dram address there.
 41  */
 42  IRAM_ATTR static void *dram_alloc_to_iram_addr(void *addr, size_t len)
 43  {
 44      uintptr_t dstart = (uintptr_t)addr; //First word
 45      uintptr_t dend = dstart + len - 4; //Last word
 46      assert(esp_ptr_in_diram_dram((void *)dstart));
 47      assert(esp_ptr_in_diram_dram((void *)dend));
 48      assert((dstart & 3) == 0);
 49      assert((dend & 3) == 0);
 50  #if SOC_DIRAM_INVERTED // We want the word before the result to hold the DRAM address
 51      uint32_t *iptr = esp_ptr_diram_dram_to_iram((void *)dend);
 52  #else
 53      uint32_t *iptr = esp_ptr_diram_dram_to_iram((void *)dstart);
 54  #endif
 55      *iptr = dstart;
 56      return iptr + 1;
 57  }
 58  
 59  
 60  static void heap_caps_alloc_failed(size_t requested_size, uint32_t caps, const char *function_name) 
 61  {
 62      if (alloc_failed_callback) {
 63          alloc_failed_callback(requested_size, caps, function_name);
 64      }
 65  
 66      #ifdef CONFIG_HEAP_ABORT_WHEN_ALLOCATION_FAILS
 67      esp_system_abort("Memory allocation failed");
 68      #endif
 69  }
 70  
 71  esp_err_t heap_caps_register_failed_alloc_callback(esp_alloc_failed_hook_t callback)
 72  {
 73      if (callback == NULL) {
 74          return ESP_ERR_INVALID_ARG;
 75      }
 76  
 77      alloc_failed_callback = callback;
 78  
 79      return ESP_OK;
 80  }
 81  
 82  bool heap_caps_match(const heap_t *heap, uint32_t caps)
 83  {
 84      return heap->heap != NULL && ((get_all_caps(heap) & caps) == caps);
 85  }
 86  
 87  /*
 88  Routine to allocate a bit of memory with certain capabilities. caps is a bitfield of MALLOC_CAP_* bits.
 89  */
 90  IRAM_ATTR void *heap_caps_malloc( size_t size, uint32_t caps )
 91  {
 92      void *ret = NULL;
 93  
 94      if (size > HEAP_SIZE_MAX) {
 95          // Avoids int overflow when adding small numbers to size, or
 96          // calculating 'end' from start+size, by limiting 'size' to the possible range
 97          heap_caps_alloc_failed(size, caps, __func__);
 98  
 99          return NULL;
100      }
101  
102      if (caps & MALLOC_CAP_EXEC) {
103          //MALLOC_CAP_EXEC forces an alloc from IRAM. There is a region which has both this as well as the following
104          //caps, but the following caps are not possible for IRAM.  Thus, the combination is impossible and we return
105          //NULL directly, even although our heap capabilities (based on soc_memory_tags & soc_memory_regions) would
106          //indicate there is a tag for this.
107          if ((caps & MALLOC_CAP_8BIT) || (caps & MALLOC_CAP_DMA)) {
108              heap_caps_alloc_failed(size, caps, __func__);
109  
110              return NULL;
111          }
112          caps |= MALLOC_CAP_32BIT; // IRAM is 32-bit accessible RAM
113      }
114  
115      if (caps & MALLOC_CAP_32BIT) {
116          /* 32-bit accessible RAM should allocated in 4 byte aligned sizes
117           * (Future versions of ESP-IDF should possibly fail if an invalid size is requested)
118           */
119          size = (size + 3) & (~3); // int overflow checked above
120      }
121  
122      for (int prio = 0; prio < SOC_MEMORY_TYPE_NO_PRIOS; prio++) {
123          //Iterate over heaps and check capabilities at this priority
124          heap_t *heap;
125          SLIST_FOREACH(heap, &registered_heaps, next) {
126              if (heap->heap == NULL) {
127                  continue;
128              }
129              if ((heap->caps[prio] & caps) != 0) {
130                  //Heap has at least one of the caps requested. If caps has other bits set that this prio
131                  //doesn't cover, see if they're available in other prios.
132                  if ((get_all_caps(heap) & caps) == caps) {
133                      //This heap can satisfy all the requested capabilities. See if we can grab some memory using it.
134                      if ((caps & MALLOC_CAP_EXEC) && esp_ptr_in_diram_dram((void *)heap->start)) {
135                          //This is special, insofar that what we're going to get back is a DRAM address. If so,
136                          //we need to 'invert' it (lowest address in DRAM == highest address in IRAM and vice-versa) and
137                          //add a pointer to the DRAM equivalent before the address we're going to return.
138                          ret = multi_heap_malloc(heap->heap, size + 4);  // int overflow checked above
139  
140                          if (ret != NULL) {
141                              return dram_alloc_to_iram_addr(ret, size + 4);  // int overflow checked above
142                          }
143                      } else {
144                          //Just try to alloc, nothing special.
145                          ret = multi_heap_malloc(heap->heap, size);
146                          if (ret != NULL) {
147                              return ret;
148                          }
149                      }
150                  }
151              }
152          }
153      }
154  
155      heap_caps_alloc_failed(size, caps, __func__);
156  
157      //Nothing usable found.
158      return NULL;
159  }
160  
161  
162  #define MALLOC_DISABLE_EXTERNAL_ALLOCS -1
163  //Dual-use: -1 (=MALLOC_DISABLE_EXTERNAL_ALLOCS) disables allocations in external memory, >=0 sets the limit for allocations preferring internal memory.
164  static int malloc_alwaysinternal_limit=MALLOC_DISABLE_EXTERNAL_ALLOCS;
165  
166  void heap_caps_malloc_extmem_enable(size_t limit)
167  {
168      malloc_alwaysinternal_limit=limit;
169  }
170  
171  /*
172   Default memory allocation implementation. Should return standard 8-bit memory. malloc() essentially resolves to this function.
173  */
174  IRAM_ATTR void *heap_caps_malloc_default( size_t size )
175  {
176      if (malloc_alwaysinternal_limit==MALLOC_DISABLE_EXTERNAL_ALLOCS) {
177          return heap_caps_malloc( size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL);
178      } else {
179          void *r;
180          if (size <= malloc_alwaysinternal_limit) {
181              r=heap_caps_malloc( size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL );
182          } else {
183              r=heap_caps_malloc( size, MALLOC_CAP_DEFAULT | MALLOC_CAP_SPIRAM );
184          }
185          if (r==NULL) {
186              //try again while being less picky
187              r=heap_caps_malloc( size, MALLOC_CAP_DEFAULT );
188          }
189          return r;
190      }
191  }
192  
193  /*
194   Same for realloc()
195   Note: keep the logic in here the same as in heap_caps_malloc_default (or merge the two as soon as this gets more complex...)
196   */
197  IRAM_ATTR void *heap_caps_realloc_default( void *ptr, size_t size )
198  {
199      if (malloc_alwaysinternal_limit==MALLOC_DISABLE_EXTERNAL_ALLOCS) {
200          return heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL );
201      } else {
202          void *r;
203          if (size <= malloc_alwaysinternal_limit) {
204              r=heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL );
205          } else {
206              r=heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT | MALLOC_CAP_SPIRAM );
207          }
208          if (r==NULL && size>0) {
209              //We needed to allocate memory, but we didn't. Try again while being less picky.
210              r=heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT );
211          }
212          return r;
213      }
214  }
215  
216  /*
217   Memory allocation as preference in decreasing order.
218   */
219  IRAM_ATTR void *heap_caps_malloc_prefer( size_t size, size_t num, ... )
220  {
221      va_list argp;
222      va_start( argp, num );
223      void *r = NULL;
224      while (num--) {
225          uint32_t caps = va_arg( argp, uint32_t );
226          r = heap_caps_malloc( size, caps );
227          if (r != NULL) {
228              break;
229          }
230      }
231      va_end( argp );
232      return r;
233  }
234  
235  /*
236   Memory reallocation as preference in decreasing order.
237   */
238  IRAM_ATTR void *heap_caps_realloc_prefer( void *ptr, size_t size, size_t num, ... )
239  {
240      va_list argp;
241      va_start( argp, num );
242      void *r = NULL;
243      while (num--) {
244          uint32_t caps = va_arg( argp, uint32_t );
245          r = heap_caps_realloc( ptr, size, caps );
246          if (r != NULL || size == 0) {
247              break;
248          }
249      }
250      va_end( argp );
251      return r;
252  }
253  
254  /*
255   Memory callocation as preference in decreasing order.
256   */
257  IRAM_ATTR void *heap_caps_calloc_prefer( size_t n, size_t size, size_t num, ... )
258  {
259      va_list argp;
260      va_start( argp, num );
261      void *r = NULL;
262      while (num--) {
263          uint32_t caps = va_arg( argp, uint32_t );
264          r = heap_caps_calloc( n, size, caps );
265          if (r != NULL) break;
266      }
267      va_end( argp );
268      return r;
269  }
270  
271  /* Find the heap which belongs to ptr, or return NULL if it's
272     not in any heap.
273  
274     (This confirms if ptr is inside the heap's region, doesn't confirm if 'ptr'
275     is an allocated block or is some other random address inside the heap.)
276  */
277  IRAM_ATTR static heap_t *find_containing_heap(void *ptr )
278  {
279      intptr_t p = (intptr_t)ptr;
280      heap_t *heap;
281      SLIST_FOREACH(heap, &registered_heaps, next) {
282          if (heap->heap != NULL && p >= heap->start && p < heap->end) {
283              return heap;
284          }
285      }
286      return NULL;
287  }
288  
289  IRAM_ATTR void heap_caps_free( void *ptr)
290  {
291      if (ptr == NULL) {
292          return;
293      }
294  
295      if (esp_ptr_in_diram_iram(ptr)) {
296          //Memory allocated here is actually allocated in the DRAM alias region and
297          //cannot be de-allocated as usual. dram_alloc_to_iram_addr stores a pointer to
298          //the equivalent DRAM address, though; free that.
299          uint32_t *dramAddrPtr = (uint32_t *)ptr;
300          ptr = (void *)dramAddrPtr[-1];
301      }
302  
303      heap_t *heap = find_containing_heap(ptr);
304      assert(heap != NULL && "free() target pointer is outside heap areas");
305      multi_heap_free(heap->heap, ptr);
306  }
307  
308  IRAM_ATTR void *heap_caps_realloc( void *ptr, size_t size, int caps)
309  {
310      bool ptr_in_diram_case = false;
311      heap_t *heap = NULL;
312      void *dram_ptr = NULL;
313      
314      if (ptr == NULL) {
315          return heap_caps_malloc(size, caps);
316      }
317  
318      if (size == 0) {
319          heap_caps_free(ptr);
320          return NULL;
321      }
322  
323      if (size > HEAP_SIZE_MAX) {
324          heap_caps_alloc_failed(size, caps, __func__);
325  
326          return NULL;
327      }
328  
329      //The pointer to memory may be aliased, we need to 
330      //recover the corresponding address before to manage a new allocation:
331      if(esp_ptr_in_diram_iram((void *)ptr)) {
332          uint32_t *dram_addr = (uint32_t *)ptr;
333          dram_ptr  = (void *)dram_addr[-1];
334          
335          heap = find_containing_heap(dram_ptr);
336          assert(heap != NULL && "realloc() pointer is outside heap areas");
337          
338          //with pointers that reside on diram space, we avoid using 
339          //the realloc implementation due to address translation issues,
340          //instead force a malloc/copy/free
341          ptr_in_diram_case = true;
342      
343      } else {
344          heap = find_containing_heap(ptr);
345          assert(heap != NULL && "realloc() pointer is outside heap areas");
346      }
347  
348      // are the existing heap's capabilities compatible with the
349      // requested ones?
350      bool compatible_caps = (caps & get_all_caps(heap)) == caps;
351  
352      if (compatible_caps && !ptr_in_diram_case) {
353          // try to reallocate this memory within the same heap
354          // (which will resize the block if it can)
355          void *r = multi_heap_realloc(heap->heap, ptr, size);
356          if (r != NULL) {
357              return r;
358          }
359      }
360  
361      // if we couldn't do that, try to see if we can reallocate
362      // in a different heap with requested capabilities.
363      void *new_p = heap_caps_malloc(size, caps);
364      if (new_p != NULL) {
365          size_t old_size = 0;
366  
367          //If we're dealing with aliased ptr, information regarding its containing
368          //heap can only be obtained with translated address.
369          if(ptr_in_diram_case) {
370              old_size = multi_heap_get_allocated_size(heap->heap, dram_ptr);
371          } else {
372              old_size = multi_heap_get_allocated_size(heap->heap, ptr);
373          }
374  
375          assert(old_size > 0);
376          memcpy(new_p, ptr, MIN(size, old_size));
377          heap_caps_free(ptr);
378          return new_p;
379      }
380  
381      heap_caps_alloc_failed(size, caps, __func__);
382  
383      return NULL;
384  }
385  
386  IRAM_ATTR void *heap_caps_calloc( size_t n, size_t size, uint32_t caps)
387  {
388      void *result;
389      size_t size_bytes;
390  
391      if (__builtin_mul_overflow(n, size, &size_bytes)) {
392          return NULL;
393      }
394  
395      result = heap_caps_malloc(size_bytes, caps);
396      if (result != NULL) {
397          bzero(result, size_bytes);
398      }
399      return result;
400  }
401  
402  size_t heap_caps_get_total_size(uint32_t caps)
403  {
404      size_t total_size = 0;
405      heap_t *heap;
406      SLIST_FOREACH(heap, &registered_heaps, next) {
407          if (heap_caps_match(heap, caps)) {
408              total_size += (heap->end - heap->start);
409          }
410      }
411      return total_size;
412  }
413  
414  size_t heap_caps_get_free_size( uint32_t caps )
415  {
416      size_t ret = 0;
417      heap_t *heap;
418      SLIST_FOREACH(heap, &registered_heaps, next) {
419          if (heap_caps_match(heap, caps)) {
420              ret += multi_heap_free_size(heap->heap);
421          }
422      }
423      return ret;
424  }
425  
426  size_t heap_caps_get_minimum_free_size( uint32_t caps )
427  {
428      size_t ret = 0;
429      heap_t *heap;
430      SLIST_FOREACH(heap, &registered_heaps, next) {
431          if (heap_caps_match(heap, caps)) {
432              ret += multi_heap_minimum_free_size(heap->heap);
433          }
434      }
435      return ret;
436  }
437  
438  size_t heap_caps_get_largest_free_block( uint32_t caps )
439  {
440      multi_heap_info_t info;
441      heap_caps_get_info(&info, caps);
442      return info.largest_free_block;
443  }
444  
445  void heap_caps_get_info( multi_heap_info_t *info, uint32_t caps )
446  {
447      bzero(info, sizeof(multi_heap_info_t));
448  
449      heap_t *heap;
450      SLIST_FOREACH(heap, &registered_heaps, next) {
451          if (heap_caps_match(heap, caps)) {
452              multi_heap_info_t hinfo;
453              multi_heap_get_info(heap->heap, &hinfo);
454  
455              info->total_free_bytes += hinfo.total_free_bytes;
456              info->total_allocated_bytes += hinfo.total_allocated_bytes;
457              info->largest_free_block = MAX(info->largest_free_block,
458                                             hinfo.largest_free_block);
459              info->minimum_free_bytes += hinfo.minimum_free_bytes;
460              info->allocated_blocks += hinfo.allocated_blocks;
461              info->free_blocks += hinfo.free_blocks;
462              info->total_blocks += hinfo.total_blocks;
463          }
464      }
465  }
466  
467  void heap_caps_print_heap_info( uint32_t caps )
468  {
469      multi_heap_info_t info;
470      printf("Heap summary for capabilities 0x%08X:\n", caps);
471      heap_t *heap;
472      SLIST_FOREACH(heap, &registered_heaps, next) {
473          if (heap_caps_match(heap, caps)) {
474              multi_heap_get_info(heap->heap, &info);
475  
476              printf("  At 0x%08x len %d free %d allocated %d min_free %d\n",
477                     heap->start, heap->end - heap->start, info.total_free_bytes, info.total_allocated_bytes, info.minimum_free_bytes);
478              printf("    largest_free_block %d alloc_blocks %d free_blocks %d total_blocks %d\n",
479                     info.largest_free_block, info.allocated_blocks,
480                     info.free_blocks, info.total_blocks);
481          }
482      }
483      printf("  Totals:\n");
484      heap_caps_get_info(&info, caps);
485  
486      printf("    free %d allocated %d min_free %d largest_free_block %d\n", info.total_free_bytes, info.total_allocated_bytes, info.minimum_free_bytes, info.largest_free_block);
487  }
488  
489  bool heap_caps_check_integrity(uint32_t caps, bool print_errors)
490  {
491      bool all_heaps = caps & MALLOC_CAP_INVALID;
492      bool valid = true;
493  
494      heap_t *heap;
495      SLIST_FOREACH(heap, &registered_heaps, next) {
496          if (heap->heap != NULL
497              && (all_heaps || (get_all_caps(heap) & caps) == caps)) {
498              valid = multi_heap_check(heap->heap, print_errors) && valid;
499          }
500      }
501  
502      return valid;
503  }
504  
505  bool heap_caps_check_integrity_all(bool print_errors)
506  {
507      return heap_caps_check_integrity(MALLOC_CAP_INVALID, print_errors);
508  }
509  
510  bool heap_caps_check_integrity_addr(intptr_t addr, bool print_errors)
511  {
512      heap_t *heap = find_containing_heap((void *)addr);
513      if (heap == NULL) {
514          return false;
515      }
516      return multi_heap_check(heap->heap, print_errors);
517  }
518  
519  void heap_caps_dump(uint32_t caps)
520  {
521      bool all_heaps = caps & MALLOC_CAP_INVALID;
522      heap_t *heap;
523      SLIST_FOREACH(heap, &registered_heaps, next) {
524          if (heap->heap != NULL
525              && (all_heaps || (get_all_caps(heap) & caps) == caps)) {
526              multi_heap_dump(heap->heap);
527          }
528      }
529  }
530  
531  void heap_caps_dump_all(void)
532  {
533      heap_caps_dump(MALLOC_CAP_INVALID);
534  }
535  
536  size_t heap_caps_get_allocated_size( void *ptr )
537  {
538      heap_t *heap = find_containing_heap(ptr);
539      size_t size = multi_heap_get_allocated_size(heap->heap, ptr);
540      return size;
541  }
542  
543  IRAM_ATTR void *heap_caps_aligned_alloc(size_t alignment, size_t size, int caps)
544  {
545      void *ret = NULL;
546  
547      if(!alignment) {
548          return NULL;
549      }
550  
551      //Alignment must be a power of two:
552      if((alignment & (alignment - 1)) != 0) {
553          return NULL;
554      }
555  
556      if (size > HEAP_SIZE_MAX) {
557          // Avoids int overflow when adding small numbers to size, or
558          // calculating 'end' from start+size, by limiting 'size' to the possible range
559          heap_caps_alloc_failed(size, caps, __func__);
560  
561          return NULL;
562      }
563  
564      //aligned alloc for now only supports default allocator or external
565      //allocator.
566      if((caps & (MALLOC_CAP_DEFAULT | MALLOC_CAP_SPIRAM)) == 0) {
567          heap_caps_alloc_failed(size, caps, __func__);
568          return NULL;
569      }
570  
571      //if caps requested are supported, clear undesired others:
572      caps &= (MALLOC_CAP_DEFAULT | MALLOC_CAP_SPIRAM);
573  
574      for (int prio = 0; prio < SOC_MEMORY_TYPE_NO_PRIOS; prio++) {
575          //Iterate over heaps and check capabilities at this priority
576          heap_t *heap;
577          SLIST_FOREACH(heap, &registered_heaps, next) {
578              if (heap->heap == NULL) {
579                  continue;
580              }
581              if ((heap->caps[prio] & caps) != 0) {
582                  //Heap has at least one of the caps requested. If caps has other bits set that this prio
583                  //doesn't cover, see if they're available in other prios.
584                  if ((get_all_caps(heap) & caps) == caps) {
585                      //Just try to alloc, nothing special.
586                      ret = multi_heap_aligned_alloc(heap->heap, size, alignment); 
587                      if (ret != NULL) {
588                          return ret;
589                      }
590                  }
591              }
592          }
593      }
594  
595      heap_caps_alloc_failed(size, caps, __func__);
596  
597      //Nothing usable found.
598      return NULL;
599  }
600  
601  IRAM_ATTR void heap_caps_aligned_free(void *ptr)
602  {
603      heap_caps_free(ptr);
604  }
605  
606  void *heap_caps_aligned_calloc(size_t alignment, size_t n, size_t size, uint32_t caps)
607  {    
608      size_t size_bytes;
609      if (__builtin_mul_overflow(n, size, &size_bytes)) {
610          return NULL;
611      }
612  
613      void *ptr = heap_caps_aligned_alloc(alignment,size_bytes, caps);
614      if(ptr != NULL) {
615          memset(ptr, 0, size_bytes);
616      }
617  
618      return ptr;
619  }