/ core / attention / __pycache__ / head_pose.cpython-314.pyc
head_pose.cpython-314.pyc
  1  +
  2  ��hi�=��,�Rt^RIt^RIt^RIt^RIt^RIHt^RIHt^RIH	t	H
  3  t
  4  Ht^RIH
t
^RIHt!RR]
4t]!R	R
  5  44tRt!RR
4t!RR4t]R8Xd�]!R4]!4tRRltRRlt]P1]4]P3]4]!R4]!R4]!R4]P54'd]P6!R4K]P;4]!R4R#R# ]dL%i;i)u9
  6  Head Pose Tracking via MacBook Camera
  7  
  8  Uses Apple's Vision framework to detect head orientation (yaw, pitch, roll)
  9  from the FaceTime camera. Combined with Tobii eye tracking, this enables
 10  multi-monitor attention detection.
 11  
 12  Architecture:
 13      MacBook Camera → Vision Framework → Head Pose → Screen Inference
 14 15      Tobii 5 → Precise Gaze (main monitor) ──────→ Attention Fusion
 16  
 17  Screen layout (configurable):
 18      [MacBook]  [Main Monitor]  [iPad]
 19         -45°         0°          +45°
 20         LEFT       CENTER       RIGHT
 21  
 22  When Tobii gaze goes invalid and head yaw < -20°, infer "looking at MacBook"
 23  When Tobii gaze goes invalid and head yaw > +20°, infer "looking at iPad"
 24  When Tobii gaze is valid, use precise coordinates on main monitor
 25  N)�	dataclass)�datetime)�Optional�List�Callable)�Enum)�Pathc�*�]tRt^"tRtRtRtRtRtRt	R#)�AttentionScreenz*Which screen the operator is attending to.�left�center�right�unknown�N)
 26  �__name__�
 27  __module__�__qualname__�__firstlineno__�__doc__�LEFT�CENTER�RIGHT�UNKNOWN�__static_attributes__r��;/Users/rcerf/repos/Sovereign_OS/core/attention/head_pose.pyr
 28  r
 29  "s��4��D�
�F��E��Grr
 30  c�ba�]tRt^*toRt]V3RlRl4tR	V3RlRlltV3RltRt	Vt
 31  R#)
 32  �HeadPosez'Head orientation from Vision framework.c� <�V^8�dQhRS[/#���return��float)�format�
__classdict__s"�r�__annotate__�HeadPose.__annotate__4s�����U�rc�(�VP^Z,#)z0Convert yaw to approximate degrees (-90 to +90).)�yaw��selfs&r�yaw_degrees�HeadPose.yaw_degrees3s���x�x�"�}�rc�,<�V^8�dQhRS[RS[RS[/#)r �left_threshold�right_thresholdr!)r#r
 33  )r$r%s"�rr&r'8s)���
*�
*��
*��
*�
 34  �	
*rc���VP'g\P#VPV8d\P#VPV8�d\P
 35  #\P#)z%Infer which screen based on head yaw.)�validr
 36  rr)rrr�r+r/r0s&&&r�infer_screen�HeadPose.infer_screen8sU���z�z�z�"�*�*�*��8�8�n�$�"�'�'�'�
�X�X��
'�"�(�(�(�"�)�)�)rc�V<�V^8�dQh/S[;R&S[;R&S[;R&S[;R&S[;R&#)r �	timestampr)�pitch�rollr2)rr#�bool)r$r%s"�rr&r'*sB�������
 37  �J�	�
 38  �L���K�
��K�rrN�gпg�?)rrrrr�propertyr,r4�__annotate_func__r�__classdictcell__�r%s@rrr*s-����1�����
*�
*��rra�
 39  import Foundation
 40  import AVFoundation
 41  import Vision
 42  
 43  class HeadPoseTracker: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
 44      let session = AVCaptureSession()
 45      let outputQueue = DispatchQueue(label: "HeadPoseOutput")
 46      var isRunning = false
 47  
 48      override init() {
 49          super.init()
 50          setupCamera()
 51      }
 52  
 53      func setupCamera() {
 54          session.sessionPreset = .medium
 55  
 56          guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front),
 57                let input = try? AVCaptureDeviceInput(device: device) else {
 58              fputs("ERROR: Could not access camera\n", stderr)
 59              return
 60          }
 61  
 62          session.addInput(input)
 63  
 64          let output = AVCaptureVideoDataOutput()
 65          output.setSampleBufferDelegate(self, queue: outputQueue)
 66          output.alwaysDiscardsLateVideoFrames = true
 67          session.addOutput(output)
 68      }
 69  
 70      func start() {
 71          isRunning = true
 72          session.startRunning()
 73      }
 74  
 75      func stop() {
 76          isRunning = false
 77          session.stopRunning()
 78      }
 79  
 80      func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
 81          guard isRunning else { return }
 82  
 83          guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
 84  
 85          let request = VNDetectFaceRectanglesRequest { [weak self] request, error in
 86              self?.handleFaceDetection(request: request, error: error)
 87          }
 88  
 89          // Enable head pose detection
 90          let faceRequest = VNDetectFaceLandmarksRequest { [weak self] request, error in
 91              self?.handleFaceLandmarks(request: request, error: error)
 92          }
 93  
 94          let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:])
 95          try? handler.perform([request, faceRequest])
 96      }
 97  
 98      func handleFaceDetection(request: VNRequest, error: Error?) {
 99          guard let observations = request.results as? [VNFaceObservation],
100                let face = observations.first else {
101              outputPose(yaw: 0, pitch: 0, roll: 0, valid: false)
102              return
103          }
104  
105          // VNFaceObservation has yaw, pitch, roll as optional NSNumber
106          let yaw = face.yaw?.floatValue ?? 0
107          let pitch = face.pitch?.floatValue ?? 0
108          let roll = face.roll?.floatValue ?? 0
109  
110          outputPose(yaw: yaw, pitch: pitch, roll: roll, valid: true)
111      }
112  
113      func handleFaceLandmarks(request: VNRequest, error: Error?) {
114          // Can be extended to get more precise eye positions
115      }
116  
117      func outputPose(yaw: Float, pitch: Float, roll: Float, valid: Bool) {
118          let timestamp = ISO8601DateFormatter().string(from: Date())
119          let json: [String: Any] = [
120              "type": "head_pose",
121              "timestamp": timestamp,
122              "yaw": yaw,
123              "pitch": pitch,
124              "roll": roll,
125              "valid": valid
126          ]
127  
128          if let data = try? JSONSerialization.data(withJSONObject: json),
129             let str = String(data: data, encoding: .utf8) {
130              print(str)
131              fflush(stdout)
132          }
133      }
134  }
135  
136  // Main
137  let tracker = HeadPoseTracker()
138  tracker.start()
139  
140  // Run for a while (will be killed by parent process)
141  RunLoop.main.run()
142  c��a�]tRt^�toRtRV3RlRlltV3RlRltRtV3RlRltV3R	lR
143  lt	V3RlRlt
144  V3R
lRltV3RlRltRt
V3RlRltRtVtR#)�HeadPoseTrackera�
145  Tracks head pose using MacBook's FaceTime camera.
146  
147  Uses Vision framework via a Swift subprocess to detect
148  face orientation (yaw, pitch, roll).
149  
150  Usage:
151      tracker = HeadPoseTracker()
152      tracker.start()
153  
154      # Get current pose
155      pose = tracker.get_pose()
156      if pose and pose.valid:
157          screen = pose.infer_screen()
158          print(f"Looking at: {screen.value}")
159  
160      tracker.stop()
161  c�&<�V^8�dQhRS[RS[/#)r r/r0r")r$r%s"�rr&�HeadPoseTracker.__annotate__�s#���N�N��N��Nrc�F�WnW nRVnRVn\P
162  !4VnRVn\P!4Vn	.Vn
163  .Vn\PVn\P !4R,R,VnR#)Nz
164  .sovereign�head_pose_tracker)r/r0�_process�_reader_thread�	threading�Event�_stop_event�
_current_pose�Lock�
165  _pose_lock�_on_pose�_on_screen_changer
166  r�_last_screenr�home�
_swift_binaryr3s&&&r�__init__�HeadPoseTracker.__init__�s~��
167  -��.��48��
�:>���$�?�?�,���15���#�.�.�*���;=��
�[]���+�3�3���"�Y�Y�[�<�7�:M�M��rc� <�V^8�dQhRS[/#r�r:)r$r%s"�rr&rC�s�����t�rc�
168  �VP4'g\R4R#\P!\	VP
169  4.\P\PR^R7VnVPP4\P!VPRRR7Vn
VPP4\R4R# \dp\RT24R	p?R#R	p?ii;i)
170  zStart head pose tracking.z1[HeadPoseTracker] Failed to compile Swift trackerFT)�stdout�stderr�text�bufsize�HeadPoseReader)�target�daemon�namez[HeadPoseTracker] Startedz#[HeadPoseTracker] Failed to start: N)�_ensure_swift_binary�print�
171  subprocess�Popen�strrR�PIPErFrJ�clearrH�Thread�_read_outputrG�start�	Exception)r+�es& rri�HeadPoseTracker.start�s����(�(�*�*��E�F��	�&�,�,��T�'�'�(�)�!���!������D�M�
���"�"�$�"+�"2�"2��(�(��%�#�D��
172  
���%�%�'��-�.����	��7��s�;�<���	�s�B7C�D�)C=�=Dc���VPP4VP'd?VPP4VPP	^R7RVnVP'd$VPP^R7RVn\R4R# \
173  PdTPP4Lzi;i)zStop head pose tracking.)�timeoutNz[HeadPoseTracker] Stopped)rJ�setrF�	terminate�waitrb�TimeoutExpired�killrG�joinrar*s&r�stop�HeadPoseTracker.stops����������=�=�=��M�M�#�#�%�
%��
�
�"�"�1�"�-�!�D�M��������$�$�Q�$�/�"&�D��
�)�*���,�,�
%��
�
�"�"�$�
%�s�B-�-/C�Cc�0<�V^8�dQhRS[S[,/#r)rr)r$r%s"�rr&rCs���&�&�(�8�,�&rc��VP;_uu_4VPuuRRR4# +'giR#;i)zGet the most recent head pose.N)rMrKr*s&r�get_pose�HeadPoseTracker.get_poses ��
�_�_�_��%�%��_�_�_�s�,�=	c� <�V^8�dQhRS[/#r�r
174  )r$r%s"�rr&rCs���'�'�O�'rc��VP4pV'd'VPVPVP4#\P
175  #)z,Get which screen the operator is looking at.)ryr4r/r0r
176  r)r+�poses& r�
177  get_screen�HeadPoseTracker.get_screens;���}�}�����$�$�T�%8�%8�$�:N�:N�O�O��&�&�&rc�6<�V^8�dQhRS[S[.R3,/#�r �callbackN)rr)r$r%s"�rr&rCs ���'�'��(��T�)9� :�'rc�<�VPPV4R#)z#Register callback for pose updates.N)rN�append�r+r�s&&r�on_pose�HeadPoseTracker.on_poses���
�
���X�&rc�:<�V^8�dQhRS[S[S[.R3,/#r�)rr
178  )r$r%s"�rr&rC"s$���0�0��?�O�2T�VZ�2Z�)[�0rc�<�VPPV4R#)z>Register callback for screen changes (old_screen, new_screen).N)rOr�r�s&&r�on_screen_change� HeadPoseTracker.on_screen_change"s�����%�%�h�/rc� <�V^8�dQhRS[/#rrV)r$r%s"�rr&rC&s���&�&�d�&rc�8�VPP4'dR#\R4VPPR4pVPPRRR7VP
\4\P!RRR\VP4RR	RR
179  RR\V4.RR^<R7pVP^8wd\R
VP24R#\R4R# \dp\RT24Rp?R#Rp?ii;i)z Compile Swift tracker if needed.Tz6[HeadPoseTracker] Compiling Swift head pose tracker...z.swift)�parents�exist_ok�swiftcz-Oz-oz
180  -framework�AVFoundation�Vision�	CoreMedia)�capture_outputrZrnz"[HeadPoseTracker] Compile failed: Fz'[HeadPoseTracker] Compiled successfullyz![HeadPoseTracker] Compile error: N)rR�existsra�with_suffix�parent�mkdir�
181  write_text�SWIFT_HEAD_POSE_TRACKERrb�runrd�
182  returncoderYrj)r+�swift_source�resultrks&   rr`�$HeadPoseTracker._ensure_swift_binary&s	�����$�$�&�&��
�F�G��)�)�5�5�h�?�����!�!�$��!�>���� 7�8�	��^�^����#�d�0�0�1� �.� �(� �+���%�� $���
�F�� � �A�%��:�6�=�=�/�J�K���;�<����	��5�a�S�9�:���	�s�<A*C5�(C5�5D�D�Dc���VPP4'g{VP'dgVPPP	4pV'g\
183  P!R4KwVPVP44K�R#R# \d:pTPP4'g\RT24Rp?R#Rp?ii;i)z*Read head pose data from Swift subprocess.g{�G�z�?z[HeadPoseTracker] Read error: N)rJ�is_setrFrX�readline�time�sleep�
_process_line�striprjra)r+�linerks&  rrh�HeadPoseTracker._read_outputNs����"�"�)�)�+�+��
�
�
�
��}�}�+�+�4�4�6����J�J�t�$���"�"�4�:�:�<�0�1>�+���
��'�'�.�.�0�0��:�1�#�>�?���
�s#�+B� B�8B�C!�(.C�C!c� <�V^8�dQhRS[/#)r r�)rd)r$r%s"�rr&rC^s���%;�%;�#�%;rc��\P!V4pVPR4R8Xd�\\P
184  !VR,P
RR44VR,VR,VR,VR	,R
185  7pVP;_uu_4W0nRRR4VPFpV!V4K	VPVPVP4pW`P8wd0VP FpV!VPV4K	W`nR#R#R# +'giL�;i \dp\RT24Rp?K�Rp?ii;i \dp\R
T24Rp?K�Rp?ii;i \P"dR#\dp\RT24Rp?R#Rp?ii;i)z'Process a JSON line from Swift tracker.�type�	head_poser7�Zz+00:00r)r8r9r2)r7r)r8r9r2Nz"[HeadPoseTracker] Callback error: z0[HeadPoseTracker] Screen change callback error: z![HeadPoseTracker] Process error: )�json�loads�getrr�
fromisoformat�replacerMrKrNrjrar4r/r0rPrO�JSONDecodeError)r+r��datar~r�rk�
186  new_screens&&     rr��HeadPoseTracker._process_line^s���#	;��:�:�d�#�D��x�x���;�.��&�4�4�T�+�5F�5N�5N�s�T\�5]�^��U���w�-��f���w�-����_�_�_�)-�&�%�!%�
�
�H�H� ���!.�"�.�.�t�/B�/B�D�DX�DX�Y�
187  ��!2�!2�2�$(�$:�$:��Z�$�T�%6�%6�
188  �C�%;�
189  )3�%�
3�+/�%�_��%�H�� B�1�#�F�G�G��H�� )�Z�!�$T�UV�TW�"X�Y�Y��Z���#�#�	���	;��5�a�S�9�:�:��	;�s��BF�D-� F�8E�A	F�
190  E'�
191  F�-D=	�8F�E$�E�F�E$�$F�'F�2F�F�F�F�G	�&G	�/G	�0G�G	)rKrPrNrOrMrFrGrJrRr/r0Nr;)rrrrrrSriruryrr�r�r`rhr�rr>r?s@rrArA�sg�����&N�N�0��>+�$&�&�
192  '�'�'�'�0�0�&�&�P� %;�%;rrAc�a�]tRtRtoRtRV3RlRlltV3RlRltV3RlR	ltV3R
193  lRltV3RlR
lt	Rt
194  VtR#)�MultiMonitorAttentioni�u
195  Fuses Tobii eye tracking with head pose for multi-monitor attention.
196  
197  When Tobii has valid gaze → use precise coordinates on main monitor
198  When Tobii gaze invalid + head left → infer MacBook attention
199  When Tobii gaze invalid + head right → infer iPad attention
200  Nc�&<�V^8�dQhRS[RS[/#)r �head_tracker�screen_names)rA�dict)r$r%s"�rr&�"MultiMonitorAttention.__annotate__�s#���K�K�%�K��Krc���WnT;'g3\PR\PR\PR/VnRVn\PVn.VnR#)�MacBookzMain Monitor�iPadFN)	r�r
201  rrrr��_tobii_valid�_current_screen�
202  _callbacks)r+r�r�s&&&rrS�MultiMonitorAttention.__init__�sa��
203  )��(�
204  �
205  �� � �)��"�"�N��!�!�6�-
206  ���"���.�5�5���HJ��rc� <�V^8�dQhRS[/#)r r2rV)r$r%s"�rr&r��s���8�8�4�8rc�8�VPpWnV'dMV'gEVPP4pV\P8wdVPV4R#R#V'g,V'd"VP\P4R#R#R#)z(Called when Tobii gaze validity changes.N)r�r�rr
207  r�_update_screenr)r+r2�	was_valid�screens&&  r�update_tobii_validity�+MultiMonitorAttention.update_tobii_validity�sm���%�%�	�!���U��&�&�1�1�3�F���0�0�0��#�#�F�+�1��u����� 6� 6�7� %�rc� <�V^8�dQhRS[/#)r r�r|)r$r%s"�rr&r��s���
208  J�
209  J�_�
210  Jrc��WP8wdJWnVPPWP4pVPFpV!W4K	R#R# \
211  dp\
RT24Rp?K3Rp?ii;i)z!Update current screen and notify.z([MultiMonitorAttention] Callback error: N)r�r�r��valuer�rjra)r+r�r_r�rks&&   rr��$MultiMonitorAttention._update_screen�sv���)�)�)�#)� ��$�$�(�(����>�D� �O�O��J��V�*�,�	*��!�J��D�Q�C�H�I�I��J�s�A�B�'A;�;Bc�:<�V^8�dQhRS[S[S[.R3,/#r�)rr
212  rd)r$r%s"�rr&r��s#���)�)��?�C�2H�$�2N�)O�)rc�<�VPPV4R#)z%Register callback for screen changes.N)r�r�r�s&&rr��&MultiMonitorAttention.on_screen_change�s�������x�(rc�6<�V^8�dQhRS[S[S[3,/#r)�tupler
213  rd)r$r%s"�rr&r��s���*�*�E�/�3�*>�$?�*rc��VPPVPVPP4pVPV3#)zGet current attention screen.)r�r�r�r�)r+r_s& r�get_current_screen�(MultiMonitorAttention.get_current_screen�s=��� � �$�$�T�%9�%9�4�;O�;O�;U�;U�V���#�#�T�)�)r)r�r�r�r�r�)N)rrrrrrSr�r�r�r�rr>r?s@rr�r��s@�����K�K� 8�8�
214  J�
215  J�)�)�*�*rr��__main__z=== Head Pose Tracker Test ===
216  c�$�V^8�dQhR\/#)r r~)r)r$s"rr&r&�s��H�H�h�Hrc��VP'd9VP4p\RVPR
RVP24R#R#)zYaw: z+.1fu° → N)r2r4rar,r�)r~r�s& rr�r��s@���:�:�:��&�&�(�F��E�$�*�*�4�0�����~�F�G�rc�0�V^8�dQhR\R\/#)r �old�newr|)r$s"rr&r&�s��J�J�o�J�O�Jrc�R�\RVPRVPR24R#)z
217  *** Screen changed: u → z ***
218  N)rar�)r�r�s&&rr�r��s"��
�&�s�y�y�k��s�y�y�k��H�IrzStarting head pose tracking...z1Turn your head left/right to see screen detectionzPress Ctrl+C to stop
219  g�������?z
220  Done.)rrbr�rHr��dataclassesrr�typingrrr�enumr�pathlibrr
221  rr�rAr�rra�trackerr�r�rir��KeyboardInterruptrurrr�<module>r�s���,����!��+�+����d���*�*��*�>h��VN;�N;�b:*�:*�z�z��	�
222  ,�-���G�H�
223  J��O�O�G�����-�.�	�
224  *�+�	�
225  =�>�	�
226  "�#��}�}���	���
227228  �3���L�L�N�	�)��9��0!�	��	�s�D	�	D�D