feed.js
1 /** 2 * linux.do unified feed — route latest/hot/top topics by site, tag, or category. 3 * 4 * Usage: 5 * linux-do feed # latest topics 6 * linux-do feed --view top --period daily # top topics (daily) 7 * linux-do feed --tag ChatGPT # latest topics by tag 8 * linux-do feed --tag 3 --view hot # hot topics by tag id 9 * linux-do feed --category 开发调优 # latest top-level category topics 10 * linux-do feed --category 94 --tag 4 --view top --period monthly 11 */ 12 import * as fs from 'node:fs'; 13 import * as os from 'node:os'; 14 import * as path from 'node:path'; 15 import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; 16 import { cli, Strategy } from '@jackwener/opencli/registry'; 17 const LINUX_DO_HOME = 'https://linux.do'; 18 const LINUX_DO_METADATA_TTL_MS = 24 * 60 * 60 * 1000; 19 let liveTagsPromise = null; 20 let liveCategoriesPromise = null; 21 let testTagOverride = null; 22 let testCategoryOverride = null; 23 let testCacheDirOverride = null; 24 /** 25 * 统一清洗名称和 slug，避免大小写与多空格影响匹配。 26 */ 27 function normalizeLookupValue(value) { 28 return value.trim().replace(/\s+/g, ' ').toLowerCase(); 29 } 30 function getHomeDir() { 31 return process.env.HOME || process.env.USERPROFILE || os.homedir(); 32 } 33 function getLinuxDoCacheDir() { 34 return testCacheDirOverride ?? path.join(getHomeDir(), '.opencli', 'cache', 'linux-do'); 35 } 36 function getMetadataCachePath(name) { 37 return path.join(getLinuxDoCacheDir(), `${name}.json`); 38 } 39 async function readMetadataCache(name) { 40 try { 41 const raw = await fs.promises.readFile(getMetadataCachePath(name), 'utf-8'); 42 const parsed = JSON.parse(raw); 43 if (!parsed || !Array.isArray(parsed.data) || typeof parsed.fetchedAt !== 'string') 44 return null; 45 const fetchedAt = new Date(parsed.fetchedAt).getTime(); 46 const fresh = Number.isFinite(fetchedAt) && (Date.now() - fetchedAt) < LINUX_DO_METADATA_TTL_MS; 47 return { data: parsed.data, fresh }; 48 } 49 catch { 50 return null; 51 } 52 } 53 async function writeMetadataCache(name, data) { 54 try { 55 const cacheDir = getLinuxDoCacheDir(); 56 await fs.promises.mkdir(cacheDir, { recursive: true }); 57 const payload = { 58 fetchedAt: new Date().toISOString(), 59 data, 60 }; 61 await fs.promises.writeFile(getMetadataCachePath(name), JSON.stringify(payload, null, 2) + '\n'); 62 } 63 catch { 64 // Cache write failures should never block command execution. 65 } 66 } 67 async function ensureLinuxDoHome(page) { 68 if (!page) 69 throw new CommandExecutionError('Browser page required'); 70 await page.goto(LINUX_DO_HOME); 71 await page.wait(2); 72 } 73 export async function fetchLinuxDoJson(page, apiPath, options = {}) { 74 if (!options.skipNavigate) { 75 await ensureLinuxDoHome(page); 76 } 77 if (!page) 78 throw new CommandExecutionError('Browser page required'); 79 const escapedPath = JSON.stringify(apiPath); 80 const result = await page.evaluate(`(async () => { 81 try { 82 const res = await fetch(${escapedPath}, { credentials: 'include' }); 83 let data = null; 84 try { data = await res.json(); } catch {} 85 return { 86 ok: res.ok, 87 status: res.status, 88 data, 89 error: data === null ? 'Response is not valid JSON' : '', 90 }; 91 } catch (error) { 92 return { 93 ok: false, 94 error: error instanceof Error ? error.message : String(error), 95 }; 96 } 97 })()`); 98 if (!result) { 99 throw new CommandExecutionError('linux.do returned an empty browser response'); 100 } 101 if (result.status === 401 || result.status === 403) { 102 throw new AuthRequiredError('linux.do', 'linux.do requires an active signed-in browser session'); 103 } 104 if (!result.ok) { 105 throw new CommandExecutionError(result.error || `linux.do request failed: HTTP ${result.status ?? 'unknown'}`); 106 } 107 if (result.error) { 108 throw new CommandExecutionError(result.error, 'Please verify your linux.do session is still valid'); 109 } 110 return result.data; 111 } 112 function findMatchingTag(records, value) { 113 const raw = value.trim(); 114 const normalized = normalizeLookupValue(value); 115 return /^\d+$/.test(raw) 116 ? records.find((item) => item.id === Number(raw)) ?? null 117 : records.find((item) => normalizeLookupValue(item.name) === normalized) 118 ?? records.find((item) => normalizeLookupValue(item.slug) === normalized) 119 ?? null; 120 } 121 function findMatchingCategory(records, value) { 122 const raw = value.trim(); 123 const normalized = normalizeLookupValue(value); 124 return /^\d+$/.test(raw) 125 ? records.find((item) => item.id === Number(raw)) ?? null 126 : records.find((item) => categoryLookupKeys(item).includes(normalized)) 127 ?? null; 128 } 129 function categoryLookupKeys(category) { 130 const keys = [category.name, category.slug]; 131 if (category.parent) { 132 keys.push(`${category.parent.name} / ${category.name}`, `${category.parent.name}/${category.name}`, `${category.parent.name}, ${category.name}`); 133 } 134 return keys.map(normalizeLookupValue); 135 } 136 function toCategoryRecord(raw, parent) { 137 return { 138 id: raw.id, 139 name: raw.name ?? '', 140 description: raw.description_text ?? raw.description ?? '', 141 slug: raw.slug ?? '', 142 parentCategoryId: parent?.id ?? null, 143 parent, 144 }; 145 } 146 async function fetchLiveTags(page) { 147 if (testTagOverride) 148 return testTagOverride; 149 if (!liveTagsPromise) { 150 liveTagsPromise = (async () => { 151 const cached = await readMetadataCache('tags'); 152 if (cached?.fresh) 153 return cached.data; 154 try { 155 const data = await fetchLinuxDoJson(page, '/tags.json', { skipNavigate: true }); 156 const tags = (Array.isArray(data?.tags) ? data.tags : []) 157 .filter((tag) => !!tag && typeof tag.id === 'number') 158 .map((tag) => ({ 159 id: tag.id, 160 slug: tag.slug ?? `${tag.id}-tag`, 161 name: tag.name ?? String(tag.id), 162 })); 163 await writeMetadataCache('tags', tags); 164 return tags; 165 } 166 catch (error) { 167 if (cached) 168 return cached.data; 169 liveTagsPromise = null; 170 throw error; 171 } 172 })().catch((error) => { 173 liveTagsPromise = null; 174 throw error; 175 }); 176 } 177 return liveTagsPromise; 178 } 179 async function fetchLiveCategories(page) { 180 if (testCategoryOverride) 181 return testCategoryOverride; 182 if (!liveCategoriesPromise) { 183 liveCategoriesPromise = (async () => { 184 const cached = await readMetadataCache('categories'); 185 if (cached?.fresh) 186 return cached.data; 187 try { 188 const data = await fetchLinuxDoJson(page, '/categories.json', { skipNavigate: true }); 189 const topCategories = Array.isArray(data?.category_list?.categories) 190 ? data.category_list.categories 191 : []; 192 const resolvedTop = topCategories.map((category) => toCategoryRecord(category, null)); 193 const parentById = new Map(resolvedTop.map((item) => [item.id, item])); 194 const subcategoryGroups = await Promise.allSettled(topCategories 195 .filter((category) => Array.isArray(category.subcategory_ids) && category.subcategory_ids.length > 0) 196 .map(async (category) => { 197 const subData = await fetchLinuxDoJson(page, `/categories.json?parent_category_id=${category.id}`, { skipNavigate: true }); 198 const subCategories = Array.isArray(subData?.category_list?.categories) 199 ? subData.category_list.categories 200 : []; 201 const parent = parentById.get(category.id) ?? null; 202 return subCategories.map((subCategory) => toCategoryRecord(subCategory, parent)); 203 })); 204 const categories = [ 205 ...resolvedTop, 206 ...subcategoryGroups.flatMap((result) => result.status === 'fulfilled' ? result.value : []), 207 ]; 208 await writeMetadataCache('categories', categories); 209 return categories; 210 } 211 catch (error) { 212 if (cached) 213 return cached.data; 214 throw error; 215 } 216 })().catch((error) => { 217 liveCategoriesPromise = null; 218 throw error; 219 }); 220 } 221 return liveCategoriesPromise; 222 } 223 function toLocalTime(utcStr) { 224 if (!utcStr) 225 return ''; 226 const d = new Date(utcStr); 227 if (isNaN(d.getTime())) 228 return utcStr; 229 return d.toLocaleString(); 230 } 231 function normalizeReplyCount(postsCount) { 232 const count = typeof postsCount === 'number' ? postsCount : 1; 233 return Math.max(0, count - 1); 234 } 235 function topicListRichFromJson(data, limit) { 236 const topics = data?.topic_list?.topics ?? []; 237 return topics.slice(0, limit).map((t) => ({ 238 title: t.fancy_title ?? t.title ?? '', 239 replies: normalizeReplyCount(t.posts_count), 240 created: toLocalTime(t.created_at), 241 likes: t.like_count ?? 0, 242 views: t.views ?? 0, 243 url: `https://linux.do/t/topic/${t.id}`, 244 })); 245 } 246 /** 247 * 解析标签，支持 id、name、slug 三种输入。 248 */ 249 async function resolveTag(page, value) { 250 const liveTag = findMatchingTag(await fetchLiveTags(page), value); 251 if (liveTag) 252 return liveTag; 253 throw new ArgumentError(`Unknown tag: ${value}`, 'Use "opencli linux-do tags" to list available tags'); 254 } 255 /** 256 * 解析分类，并补齐父分类信息。 257 */ 258 async function resolveCategory(page, value) { 259 const liveCategory = findMatchingCategory(await fetchLiveCategories(page), value); 260 if (liveCategory) 261 return liveCategory; 262 throw new ArgumentError(`Unknown category: ${value}`, 'Use "opencli linux-do categories" to list available categories'); 263 } 264 /** 265 * 将命令参数转换为最终请求地址 266 */ 267 async function resolveFeedRequest(page, kwargs) { 268 const view = (kwargs.view || 'latest'); 269 const period = (kwargs.period || 'weekly'); 270 if (kwargs.period && view !== 'top') { 271 throw new ArgumentError('--period is only valid with --view top'); 272 } 273 const params = new URLSearchParams(); 274 if (kwargs.order && kwargs.order !== 'default') 275 params.set('order', kwargs.order); 276 if (kwargs.ascending) 277 params.set('ascending', 'true'); 278 if (kwargs.limit) 279 params.set('per_page', String(kwargs.limit)); 280 const tagValue = typeof kwargs.tag === 'string' ? kwargs.tag.trim() : ''; 281 const categoryValue = typeof kwargs.category === 'string' ? kwargs.category.trim() : ''; 282 if (!tagValue && !categoryValue) { 283 const query = new URLSearchParams(params); 284 if (view === 'top') 285 query.set('period', period); 286 const jsonSuffix = query.toString() ? `?${query.toString()}` : ''; 287 return { 288 url: `${view === 'latest' ? '/latest.json' : view === 'hot' ? '/hot.json' : '/top.json'}${jsonSuffix}`, 289 }; 290 } 291 const tag = tagValue ? await resolveTag(page, tagValue) : null; 292 const category = categoryValue ? await resolveCategory(page, categoryValue) : null; 293 const categorySegments = category 294 ? (category.parent 295 ? [category.parent.slug, category.slug, String(category.id)] 296 : [category.slug, String(category.id)]) 297 .map(encodeURIComponent) 298 .join('/') 299 : ''; 300 const tagSegment = tag ? `${encodeURIComponent(tag.slug || `${tag.id}-tag`)}/${tag.id}` : ''; 301 const basePath = category && tag 302 ? `/tags/c/${categorySegments}/${tagSegment}` 303 : category 304 ? `/c/${categorySegments}` 305 : `/tag/${tagSegment}`; 306 const query = new URLSearchParams(params); 307 if (view === 'top') 308 query.set('period', period); 309 const jsonSuffix = query.toString() ? `?${query.toString()}` : ''; 310 return { 311 url: `${basePath}${view === 'latest' ? '.json' : `/l/${view}.json`}${jsonSuffix}`, 312 }; 313 } 314 export const LINUX_DO_FEED_ARGS = [ 315 { 316 name: 'view', 317 type: 'str', 318 default: 'latest', 319 help: 'View type', 320 choices: ['latest', 'hot', 'top'], 321 }, 322 { 323 name: 'tag', 324 type: 'str', 325 help: 'Tag name, slug, or id', 326 }, 327 { 328 name: 'category', 329 type: 'str', 330 help: 'Category name, slug, id, or parent/name path', 331 }, 332 { name: 'limit', type: 'int', default: 20, help: 'Number of items (per_page)' }, 333 { 334 name: 'order', 335 type: 'str', 336 default: 'default', 337 help: 'Sort order', 338 choices: [ 339 'default', 340 'created', 341 'activity', 342 'views', 343 'posts', 344 'category', 345 'likes', 346 'op_likes', 347 'posters', 348 ], 349 }, 350 { name: 'ascending', type: 'boolean', default: false, help: 'Sort ascending (default: desc)' }, 351 { 352 name: 'period', 353 type: 'str', 354 help: 'Time period (only for --view top)', 355 choices: ['all', 'daily', 'weekly', 'monthly', 'quarterly', 'yearly'], 356 }, 357 ]; 358 export async function executeLinuxDoFeed(page, kwargs) { 359 const limit = (kwargs.limit || 20); 360 await ensureLinuxDoHome(page); 361 const request = await resolveFeedRequest(page, kwargs); 362 const data = await fetchLinuxDoJson(page, request.url, { skipNavigate: true }); 363 return topicListRichFromJson(data, limit); 364 } 365 export function buildLinuxDoCompatFooter(replacement) { 366 return `Deprecated compatibility command. Prefer: ${replacement}`; 367 } 368 cli({ 369 site: 'linux-do', 370 name: 'feed', 371 description: 'linux.do 话题列表（需登录；支持全站、标签、分类）', 372 domain: 'linux.do', 373 strategy: Strategy.COOKIE, 374 browser: true, 375 columns: ['title', 'replies', 'created', 'likes', 'views', 'url'], 376 args: LINUX_DO_FEED_ARGS, 377 func: executeLinuxDoFeed, 378 }); 379 export const __test__ = { 380 resetMetadataCaches() { 381 liveTagsPromise = null; 382 liveCategoriesPromise = null; 383 testTagOverride = null; 384 testCategoryOverride = null; 385 testCacheDirOverride = null; 386 }, 387 setLiveMetadataForTests({ tags, categories, }) { 388 liveTagsPromise = null; 389 liveCategoriesPromise = null; 390 testTagOverride = tags ?? null; 391 testCategoryOverride = categories ?? null; 392 }, 393 setCacheDirForTests(dir) { 394 testCacheDirOverride = dir; 395 }, 396 resolveFeedRequest, 397 };