1  #if !defined(__COREFOUNDATION_CFSTRINGTOKENIZER__)
 2  #define __COREFOUNDATION_CFSTRINGTOKENIZER__ 1
 3  
 4  #include <CoreFoundation/CFBase.h>
 5  #include <CoreFoundation/CFLocale.h>
 6  #include <CoreFoundation/CFString.h>
 7  
 8  CF_IMPLICIT_BRIDGING_ENABLED
 9  CF_EXTERN_C_BEGIN
10  
11  typedef struct __CFStringTokenizer *CFStringTokenizerRef;
12  
13  enum {
14      kCFStringTokenizerUnitWord                     = 0,
15      kCFStringTokenizerUnitSentence                 = 1,
16      kCFStringTokenizerUnitParagraph                = 2,
17      kCFStringTokenizerUnitLineBreak                = 3,
18      kCFStringTokenizerUnitWordBoundary             = 4,
19      kCFStringTokenizerAttributeLatinTranscription  = 1UL << 16,
20      kCFStringTokenizerAttributeLanguage            = 1UL << 17,
21  };
22  
23  typedef CF_OPTIONS(CFOptionFlags, CFStringTokenizerTokenType) {
24      kCFStringTokenizerTokenNone                    = 0,
25      kCFStringTokenizerTokenNormal                  = 1UL << 0,
26      kCFStringTokenizerTokenHasSubTokensMask        = 1UL << 1,
27      kCFStringTokenizerTokenHasDerivedSubTokensMask = 1UL << 2,
28      kCFStringTokenizerTokenHasHasNumbersMask       = 1UL << 3,
29      kCFStringTokenizerTokenHasNonLettersMask       = 1UL << 4,
30      kCFStringTokenizerTokenIsCJWordMask            = 1UL << 5
31  };
32  
33  CF_EXPORT CFStringRef CFStringTokenizerCopyBestStringLanguage(CFStringRef string, CFRange range);
34  CF_EXPORT CFTypeID CFStringTokenizerGetTypeID(void);
35  CF_EXPORT CFStringTokenizerRef CFStringTokenizerCreate(CFAllocatorRef alloc, CFStringRef string, CFRange range, CFOptionFlags options, CFLocaleRef locale);
36  CF_EXPORT void CFStringTokenizerSetString(CFStringTokenizerRef tokenizer, CFStringRef string, CFRange range);
37  CF_EXPORT CFStringTokenizerTokenType CFStringTokenizerGoToTokenAtIndex(CFStringTokenizerRef tokenizer, CFIndex index);
38  CF_EXPORT CFStringTokenizerTokenType CFStringTokenizerAdvanceToNextToken(CFStringTokenizerRef tokenizer);
39  CF_EXPORT CFRange CFStringTokenizerGetCurrentTokenRange(CFStringTokenizerRef tokenizer);
40  CF_EXPORT CFTypeRef CFStringTokenizerCopyCurrentTokenAttribute(CFStringTokenizerRef tokenizer, CFOptionFlags attribute);
41  CF_EXPORT CFIndex CFStringTokenizerGetCurrentSubTokens(CFStringTokenizerRef tokenizer, CFRange *ranges, CFIndex maxRangeLength, CFMutableArrayRef derivedSubTokens);
42  
43  CF_EXTERN_C_END
44  CF_IMPLICIT_BRIDGING_DISABLED
45  
46  #endif /* ! __COREFOUNDATION_CFSTRINGTOKENIZER__ */