1  /*
  2   * Copyright (c) 2020 Apple Inc. All rights reserved.
  3   *
  4   * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  5   *
  6   * This file contains Original Code and/or Modifications of Original Code
  7   * as defined in and that are subject to the Apple Public Source License
  8   * Version 2.0 (the 'License'). You may not use this file except in
  9   * compliance with the License. The rights granted to you under the License
 10   * may not be used to create, or enable the creation or redistribution of,
 11   * unlawful or unlicensed copies of an Apple operating system, or to
 12   * circumvent, violate, or enable the circumvention or violation of, any
 13   * terms of an Apple operating system software license agreement.
 14   *
 15   * Please obtain a copy of the License at
 16   * http://www.opensource.apple.com/apsl/ and read it before using this file.
 17   *
 18   * The Original Code and all software distributed under the License are
 19   * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 20   * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 21   * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 22   * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 23   * Please see the License for the specific language governing rights and
 24   * limitations under the License.
 25   *
 26   * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 27   */
 28  
 29  #ifndef _KERN_PERCPU_H_
 30  #define _KERN_PERCPU_H_
 31  
 32  #include <mach/vm_types.h>
 33  
 34  __BEGIN_DECLS
 35  
 36  #if XNU_KERNEL_PRIVATE
 37  #include <libkern/section_keywords.h>
 38  #include <os/atomic_private.h>
 39  
 40  #pragma GCC visibility push(hidden)
 41  
 42  /*!
 43   * @macro PERCPU_DECL
 44   *
 45   * @abstract
 46   * Declares a per-CPU variable in a header.
 47   *
 48   * @param type_t        the per-CPU variable type
 49   * @param name          the per-CPU variable name
 50   */
 51  #define PERCPU_DECL(type_t, name) \
 52  	extern type_t __PERCPU_NAME(name)
 53  
 54  /*!
 55   * @macro PERCPU_DATA
 56   *
 57   * @abstract
 58   * Defines a per-CPU variable in a translation unit.
 59   *
 60   * @discussion
 61   * @c PERCPU_DECL can be used in headers to export the variable to clients.
 62   *
 63   * By default, per-cpu data is 0-initialized. Per-CPU data is allocated during
 64   * the STARTUP_SUB_KMEM_ALLOC phase and can be initialized with a STARTUP
 65   * callback in any later phase.
 66   *
 67   * Usage is:
 68   * <code>
 69   *   [ static ] type PERCPU_DATA(name);
 70   * </code>
 71   *
 72   * @param name          the per-CPU variable name
 73   */
 74  #define PERCPU_DATA(name) \
 75  	__percpu __PERCPU_NAME(name) = {0}
 76  
 77  /*!
 78   * @macro PERCPU_GET
 79   *
 80   * @abstract
 81   * Gets a pointer to the per-CPU instance of the variable for the processor the
 82   * code is currently running on.
 83   *
 84   * @discussion
 85   * It is expected that preemption or interrupts are disabled when this is used,
 86   * as a context-switch might move the current thread to another CPU.
 87   *
 88   * It is also valid in code that wasn't already disabling preemption and cares
 89   * about code-gen size a lot to use this outside of a preemption-disabled
 90   * section provided that the data is modified using atomics.
 91   *
 92   * Note that if several per-CPU pointers are acquired in short succession,
 93   * @c PERCPU_GET_WITH_BASE can be used to avoid the repeated calls to
 94   * @c current_percpu_base() which the compiler wont't elide.
 95   *
 96   * @param name          the per-CPU variable name
 97   */
 98  #define PERCPU_GET(name) \
 99  	__PERCPU_CAST(name, current_percpu_base() + __PERCPU_ADDR(name))
100  
101  /*!
102   * @function current_percpu_base()
103   *
104   * @abstract
105   * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE().
106   *
107   * @see PERCPU_GET() for conditions of use.
108   */
109  extern vm_offset_t current_percpu_base(void);
110  
111  /*!
112   * @macro PERCPU_GET_MASTER
113   *
114   * @abstract
115   * Gets a pointer to the master per-CPU instance of the variable.
116   *
117   * @param base          the per-CPU base to use
118   * @param name          the per-CPU variable name
119   */
120  #define PERCPU_GET_MASTER(name) \
121  	(&__PERCPU_NAME(name))
122  
123  /*!
124   * @macro PERCPU_GET_WITH_BASE
125   *
126   * @abstract
127   * Gets a pointer to the per-CPU instance of the variable for the specified
128   * base.
129   *
130   * @param base          the per-CPU base to use
131   * @param name          the per-CPU variable name
132   */
133  #define PERCPU_GET_WITH_BASE(base, name) \
134  	__PERCPU_CAST(name, base + __PERCPU_ADDR(name))
135  
136  /*!
137   * @macro PERCPU_GET_RELATIVE
138   *
139   * @abstract
140   * Gets a pointer to the per-CPU instance of a variable relative to another
141   * known one.
142   *
143   * @description
144   * When a per-CPU slot address is known, but the caller doesn't know the base
145   * from which it was derived, then this allows to compute another per-CPU slot
146   * address for a different variable but for the same CPU, without any loads.
147   *
148   * @param name          the per-CPU variable name
149   * @param other         the other per-CPU variable name
150   * @param ptr           a pointer to the other variable slot
151   */
152  #define PERCPU_GET_RELATIVE(name, other, ptr) ({ \
153  	__PERCPU_TYPE(other) __other_ptr = (ptr); /* type check */ \
154  	vm_offset_t __offs = __PERCPU_ADDR(name) - __PERCPU_ADDR(other); \
155  	__PERCPU_CAST(name, (vm_address_t)__other_ptr + __offs); \
156  })
157  
158  /*!
159   * @macro percpu_foreach_base()
160   *
161   * @abstract
162   * Enumerates all Per-CPU variable bases.
163   *
164   * @param it            the name of the iterator
165   */
166  #define percpu_foreach_base(it) \
167  	for (vm_offset_t it = 0, \
168  	    __next_ ## it = percpu_base.start, \
169  	    __end_ ## it = percpu_base.end; \
170          \
171  	    it <= __end_ ## it; \
172          \
173  	    it = __next_ ## it, \
174  	    __next_ ## it += percpu_section_size())
175  
176  /*!
177   * @macro percpu_foreach()
178   *
179   * @abstract
180   * Enumerates all Per-CPU variable instances.
181   *
182   * @param it            the name of the iterator
183   * @param name          the per-CPU variable name
184   */
185  #define percpu_foreach(it, name) \
186  	for (__PERCPU_TYPE(name) it, \
187  	    __base_ ## it = NULL, \
188  	    __next_ ## it = (typeof(it))percpu_base.start, \
189  	    __end_ ## it = (typeof(it))percpu_base.end; \
190          \
191  	    (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
192  	    __base_ ## it <= __end_ ## it); \
193          \
194  	    __base_ ## it = __next_ ## it, \
195  	    __next_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
196  
197  /*!
198   * @macro percpu_foreach_secondary_base()
199   *
200   * @abstract
201   * Enumerates all Per-CPU variable bases, skipping the master slot.
202   *
203   * @param it            the name of the iterator
204   */
205  #define percpu_foreach_secondary_base(it) \
206  	for (vm_offset_t it = percpu_base.start, __end_ ## it = percpu_base.end; \
207  	    it <= __end_ ## it; it += percpu_section_size())
208  
209  /*!
210   * @macro percpu_foreach_secondary()
211   *
212   * @abstract
213   * Enumerates all Per-CPU variable instances, skipping the master slot.
214   *
215   * @param it            the name of the iterator
216   * @param name          the per-CPU variable name
217   */
218  #define percpu_foreach_secondary(it, name) \
219  	for (__PERCPU_TYPE(name) it, \
220  	    __base_ ## it = (typeof(it))percpu_base.start, \
221  	    __end_ ## it = (typeof(it))percpu_base.end; \
222          \
223  	    (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
224  	    __base_ ## it <= __end_ ## it); \
225          \
226  	    __base_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
227  
228  #pragma mark - implementation details
229  
230  /*
231   * Below this point are implementation details that should not be used directly,
232   * except by the macros above, or architecture specific code.
233   */
234  
235  #define __percpu                        __attribute__((section("__DATA, __percpu")))
236  #define __PERCPU_NAME(name)             percpu_slot_ ## name
237  #define __PERCPU_ADDR(name)             ((vm_offset_t)&__PERCPU_NAME(name))
238  #define __PERCPU_TYPE(name)             typeof(&__PERCPU_NAME(name))
239  #define __PERCPU_CAST(name, expr)       ((__PERCPU_TYPE(name))(expr))
240  
241  /*
242   * Note for implementors:
243   *
244   * A `base` represents a pointer in the percpu allocation offset by
245   * `percpu_section_start()` so that PERCPU_GET() is a single addition.
246   *
247   * percpu_base.end is inclusive, so that percpu_foreach() and
248   * percpu_foreach_base() can do a `<=` comparison.
249   *
250   * Because the first base is `0` (because the master CPU is using the static
251   * percpu section), it allows for the compiler to know that for the first
252   * iteration the comparison is always true.
253   */
254  extern struct percpu_base {
255  	vm_address_t start;
256  	vm_address_t end;
257  	vm_offset_t  size;
258  } percpu_base;
259  
260  static __pure2 inline vm_offset_t
261  percpu_section_start(void)
262  {
263  	extern char __percpu_section_start[] __SECTION_START_SYM("__DATA", "__percpu");
264  	return (vm_offset_t)__percpu_section_start;
265  }
266  
267  static __pure2 inline vm_offset_t
268  percpu_section_end(void)
269  {
270  	extern char __percpu_section_end[] __SECTION_END_SYM("__DATA", "__percpu");
271  	return (vm_offset_t)__percpu_section_end;
272  }
273  
274  static __pure2 inline vm_size_t
275  percpu_section_size(void)
276  {
277  	return percpu_section_end() - percpu_section_start();
278  }
279  
280  #pragma GCC visibility pop
281  #endif /* XNU_KERNEL_PRIVATE */
282  
283  __END_DECLS
284  
285  #endif /* _KERN_PERCPU_H_ */