timeline.js
1 import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; 2 import { cli, Strategy } from '@jackwener/opencli/registry'; 3 import { resolveTwitterQueryId, extractMedia } from './shared.js'; 4 // ── Twitter GraphQL constants ────────────────────────────────────────── 5 const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; 6 const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ'; 7 const HOME_LATEST_TIMELINE_QUERY_ID = 'BKB7oi212Fi7kQtCBGE4zA'; 8 // Endpoint config: for-you uses GET HomeTimeline, following uses POST HomeLatestTimeline 9 const TIMELINE_ENDPOINTS = { 10 'for-you': { endpoint: 'HomeTimeline', method: 'GET', fallbackQueryId: HOME_TIMELINE_QUERY_ID }, 11 following: { endpoint: 'HomeLatestTimeline', method: 'POST', fallbackQueryId: HOME_LATEST_TIMELINE_QUERY_ID }, 12 }; 13 const FEATURES = { 14 rweb_video_screen_enabled: false, 15 profile_label_improvements_pcf_label_in_post_enabled: true, 16 rweb_tipjar_consumption_enabled: true, 17 verified_phone_label_enabled: false, 18 creator_subscriptions_tweet_preview_api_enabled: true, 19 responsive_web_graphql_timeline_navigation_enabled: true, 20 responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, 21 premium_content_api_read_enabled: false, 22 communities_web_enable_tweet_community_results_fetch: true, 23 c9s_tweet_anatomy_moderator_badge_enabled: true, 24 responsive_web_grok_analyze_button_fetch_trends_enabled: false, 25 responsive_web_grok_analyze_post_followups_enabled: true, 26 responsive_web_jetfuel_frame: false, 27 responsive_web_grok_share_attachment_enabled: true, 28 articles_preview_enabled: true, 29 responsive_web_edit_tweet_api_enabled: true, 30 graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, 31 view_counts_everywhere_api_enabled: true, 32 longform_notetweets_consumption_enabled: true, 33 responsive_web_twitter_article_tweet_consumption_enabled: true, 34 tweet_awards_web_tipping_enabled: false, 35 responsive_web_grok_show_grok_translated_post: false, 36 responsive_web_grok_analysis_button_from_backend: false, 37 creator_subscriptions_quote_tweet_preview_enabled: false, 38 freedom_of_speech_not_reach_fetch_enabled: true, 39 standardized_nudges_misinfo: true, 40 tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, 41 longform_notetweets_rich_text_read_enabled: true, 42 longform_notetweets_inline_media_enabled: true, 43 responsive_web_grok_image_annotation_enabled: true, 44 responsive_web_enhance_cards_enabled: false, 45 }; 46 function buildTimelineVariables(type, count, cursor) { 47 const vars = { 48 count, 49 includePromotedContent: false, 50 latestControlAvailable: true, 51 requestContext: 'launch', 52 }; 53 if (type === 'for-you') 54 vars.withCommunity = true; 55 if (type === 'following') 56 vars.seenTweetIds = []; 57 if (cursor) 58 vars.cursor = cursor; 59 return vars; 60 } 61 function buildHomeTimelineUrl(queryId, endpoint, vars) { 62 return (`/i/api/graphql/${queryId}/${endpoint}` + 63 `?variables=${encodeURIComponent(JSON.stringify(vars))}` + 64 `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`); 65 } 66 function extractTweet(result, seen) { 67 if (!result) 68 return null; 69 const tw = result.tweet || result; 70 const l = tw.legacy || {}; 71 if (!tw.rest_id || seen.has(tw.rest_id)) 72 return null; 73 seen.add(tw.rest_id); 74 const u = tw.core?.user_results?.result; 75 const screenName = u?.legacy?.screen_name || u?.core?.screen_name || 'unknown'; 76 const noteText = tw.note_tweet?.note_tweet_results?.result?.text; 77 const views = tw.views?.count ? parseInt(tw.views.count, 10) : 0; 78 return { 79 id: tw.rest_id, 80 author: screenName, 81 text: noteText || l.full_text || '', 82 likes: l.favorite_count || 0, 83 retweets: l.retweet_count || 0, 84 replies: l.reply_count || 0, 85 views, 86 created_at: l.created_at || '', 87 url: `https://x.com/${screenName}/status/${tw.rest_id}`, 88 ...extractMedia(l), 89 }; 90 } 91 function parseHomeTimeline(data, seen) { 92 const tweets = []; 93 let nextCursor = null; 94 // Both HomeTimeline and HomeLatestTimeline share the same response envelope 95 const instructions = data?.data?.home?.home_timeline_urt?.instructions || []; 96 for (const inst of instructions) { 97 for (const entry of inst.entries || []) { 98 const c = entry.content; 99 // Cursor entries 100 if (c?.entryType === 'TimelineTimelineCursor' || c?.__typename === 'TimelineTimelineCursor') { 101 if (c.cursorType === 'Bottom') 102 nextCursor = c.value; 103 continue; 104 } 105 if (entry.entryId?.startsWith('cursor-bottom-')) { 106 nextCursor = c?.value || nextCursor; 107 continue; 108 } 109 // Single tweet entry 110 const tweetResult = c?.itemContent?.tweet_results?.result; 111 if (tweetResult) { 112 // Skip promoted content 113 if (c?.itemContent?.promotedMetadata) 114 continue; 115 const tw = extractTweet(tweetResult, seen); 116 if (tw) 117 tweets.push(tw); 118 continue; 119 } 120 // Conversation module (grouped tweets) 121 for (const item of c?.items || []) { 122 const nested = item.item?.itemContent?.tweet_results?.result; 123 if (nested) { 124 if (item.item?.itemContent?.promotedMetadata) 125 continue; 126 const tw = extractTweet(nested, seen); 127 if (tw) 128 tweets.push(tw); 129 } 130 } 131 } 132 } 133 return { tweets, nextCursor }; 134 } 135 // ── CLI definition ──────────────────────────────────────────────────── 136 cli({ 137 site: 'twitter', 138 name: 'timeline', 139 description: 'Fetch Twitter timeline (for-you or following)', 140 domain: 'x.com', 141 strategy: Strategy.COOKIE, 142 browser: true, 143 args: [ 144 { 145 name: 'type', 146 default: 'for-you', 147 choices: ['for-you', 'following'], 148 help: 'Timeline type: for-you (algorithmic) or following (chronological)', 149 }, 150 { name: 'limit', type: 'int', default: 20 }, 151 ], 152 columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url', 'has_media', 'media_urls'], 153 func: async (page, kwargs) => { 154 const limit = kwargs.limit || 20; 155 const timelineType = kwargs.type === 'following' ? 'following' : 'for-you'; 156 const { endpoint, method, fallbackQueryId } = TIMELINE_ENDPOINTS[timelineType]; 157 // Navigate to x.com for cookie context 158 await page.goto('https://x.com'); 159 await page.wait(3); 160 // Extract CSRF token 161 const ct0 = await page.evaluate(`() => { 162 return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null; 163 }`); 164 if (!ct0) 165 throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); 166 // Dynamically resolve queryId for the selected endpoint 167 const queryId = await resolveTwitterQueryId(page, endpoint, fallbackQueryId); 168 // Build auth headers 169 const headers = JSON.stringify({ 170 Authorization: `Bearer ${decodeURIComponent(BEARER_TOKEN)}`, 171 'X-Csrf-Token': ct0, 172 'X-Twitter-Auth-Type': 'OAuth2Session', 173 'X-Twitter-Active-User': 'yes', 174 }); 175 // Paginate — fetch in browser, parse in TypeScript 176 const allTweets = []; 177 const seen = new Set(); 178 let cursor = null; 179 for (let i = 0; i < 5 && allTweets.length < limit; i++) { 180 const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering 181 const variables = buildTimelineVariables(timelineType, fetchCount, cursor); 182 const apiUrl = buildHomeTimelineUrl(queryId, endpoint, variables); 183 const data = await page.evaluate(`async () => { 184 const r = await fetch("${apiUrl}", { method: "${method}", headers: ${headers}, credentials: 'include' }); 185 return r.ok ? await r.json() : { error: r.status }; 186 }`); 187 if (data?.error) { 188 if (allTweets.length === 0) 189 throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`); 190 break; 191 } 192 const { tweets, nextCursor } = parseHomeTimeline(data, seen); 193 allTweets.push(...tweets); 194 if (!nextCursor || nextCursor === cursor) 195 break; 196 cursor = nextCursor; 197 } 198 return allTweets.slice(0, limit); 199 }, 200 }); 201 export const __test__ = { 202 buildTimelineVariables, 203 buildHomeTimelineUrl, 204 parseHomeTimeline, 205 };