1  #pragma once
  2  
  3  // ggml-backend internal header
  4  
  5  #include "ggml-backend.h"
  6  
  7  #ifdef  __cplusplus
  8  extern "C" {
  9  #endif
 10  
 11      //
 12      // Backend buffer
 13      //
 14  
 15      // buffer type
 16      typedef void * ggml_backend_buffer_type_context_t;
 17  
 18      struct ggml_backend_buffer_type_i {
 19          const char *          (*GGML_CALL get_name)        (ggml_backend_buffer_type_t buft);
 20          // allocate a buffer of this type
 21          ggml_backend_buffer_t (*GGML_CALL alloc_buffer)    (ggml_backend_buffer_type_t buft, size_t size);
 22          // tensor alignment
 23          size_t                (*GGML_CALL get_alignment)   (ggml_backend_buffer_type_t buft);
 24          // max buffer size that can be allocated
 25          size_t                (*GGML_CALL get_max_size)    (ggml_backend_buffer_type_t buft);
 26          // data size needed to allocate the tensor, including padding
 27          size_t                (*GGML_CALL get_alloc_size)  (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
 28          // check if tensor data is in host memory
 29          bool                  (*GGML_CALL is_host)         (ggml_backend_buffer_type_t buft);
 30      };
 31  
 32      struct ggml_backend_buffer_type {
 33          struct ggml_backend_buffer_type_i  iface;
 34          ggml_backend_buffer_type_context_t context;
 35      };
 36  
 37      // buffer
 38      typedef void * ggml_backend_buffer_context_t;
 39  
 40      struct ggml_backend_buffer_i {
 41          const char * (*GGML_CALL get_name)   (ggml_backend_buffer_t buffer);
 42          void         (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer);
 43          void *       (*GGML_CALL get_base)   (ggml_backend_buffer_t buffer);
 44          void         (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
 45          void         (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
 46          void         (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
 47          bool         (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
 48          void         (*GGML_CALL clear)      (ggml_backend_buffer_t buffer, uint8_t value);
 49          void         (*GGML_CALL reset)      (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
 50      };
 51  
 52      struct ggml_backend_buffer {
 53          struct ggml_backend_buffer_i  iface;
 54          ggml_backend_buffer_type_t    buft;
 55          ggml_backend_buffer_context_t context;
 56          size_t size;
 57          enum ggml_backend_buffer_usage usage;
 58      };
 59  
 60      GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
 61                     ggml_backend_buffer_type_t      buft,
 62              struct ggml_backend_buffer_i           iface,
 63                     ggml_backend_buffer_context_t   context,
 64                     size_t                          size);
 65  
 66      // do not use directly, use ggml_backend_tensor_copy instead
 67      bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
 68  
 69      // buffer that contains a collection of buffers
 70      GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
 71      GGML_CALL bool                  ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
 72      GGML_CALL void                  ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
 73  
 74      //
 75      // Backend
 76      //
 77  
 78      typedef void * ggml_backend_context_t;
 79  
 80      struct ggml_backend_i {
 81          const char * (*GGML_CALL get_name)(ggml_backend_t backend);
 82  
 83          void (*GGML_CALL free)(ggml_backend_t backend);
 84  
 85          // buffer allocation
 86          ggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend);
 87  
 88          // (optional) asynchronous tensor data access
 89          void (*GGML_CALL set_tensor_async)(ggml_backend_t backend,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
 90          void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
 91          bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
 92  
 93          // (optional) complete all pending operations
 94          void (*GGML_CALL synchronize)(ggml_backend_t backend);
 95  
 96          // compute graph with a plan (not used currently)
 97          // create a new plan for a graph
 98          ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);
 99          void                      (*GGML_CALL graph_plan_free)   (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
100          // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology
101          void                      (*GGML_CALL graph_plan_update) (ggml_backend_t backend, ggml_backend_graph_plan_t plan, const struct ggml_cgraph * cgraph);
102          // compute the graph with the plan
103          enum ggml_status          (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);
104  
105          // compute graph without a plan (async)
106          enum ggml_status (*GGML_CALL graph_compute)     (ggml_backend_t backend, struct ggml_cgraph * cgraph);
107  
108          // check if the backend can compute an operation
109          bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);
110  
111          // check if the backend can use tensors allocated in a buffer type
112          bool (*GGML_CALL supports_buft)(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
113  
114          // check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer
115          // these should be expensive operations with large batch sizes that may benefit from running on this backend
116          // even if the weight has to be copied from the CPU temporarily
117          bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op);
118  
119          // (optional) event synchronization
120          // create a new event that can record events on this backend instance
121          ggml_backend_event_t (*GGML_CALL event_new)         (ggml_backend_t backend);
122          void                 (*GGML_CALL event_free)        (ggml_backend_event_t event);
123          // record an event on the backend instance that created it
124          void                 (*GGML_CALL event_record)      (ggml_backend_event_t event);
125          // wait for an event on on a different backend instance
126          void                 (*GGML_CALL event_wait)        (ggml_backend_t backend, ggml_backend_event_t event);
127          // block until an event is recorded
128          void                 (*GGML_CALL event_synchronize) (ggml_backend_event_t event);
129      };
130  
131      struct ggml_backend {
132          ggml_guid_t guid;
133  
134          struct ggml_backend_i iface;
135          ggml_backend_context_t context;
136      };
137  
138      struct ggml_backend_event {
139          ggml_backend_t backend;
140          void * context;
141      };
142  
143      //
144      // Backend registry
145      //
146  
147      typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data);
148  
149      GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);
150  
151  #ifdef  __cplusplus
152  }
153  #endif