Skip to content

Add reliable CJK bookmark search #179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
willzhqiang wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add reliable CJK bookmark search #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 62 additions & 6 deletions src/bookmarks-db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,13 @@ function buildBookmarkWhereClause(filters: BookmarkTimelineFilters): {
const params: Array<string | number> = [];

if (filters.query) {
conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`);
params.push(filters.query);
if (isCjkQuery(filters.query)) {
conditions.push(cjkLikeCondition('b', filters.query));
params.push(...cjkLikeParams(filters.query));
} else {
conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`);
params.push(filters.query);
}
}
if (filters.author) {
conditions.push(`b.author_handle = ? COLLATE NOCASE`);
Expand Down Expand Up @@ -536,19 +541,70 @@ export function sanitizeFtsQuery(query: string): string {
.join(' ');
}

function isCjkQuery(query: string): boolean {
return /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uac00-\ud7af]/u.test(query);
}

function escapeLikeQuery(query: string): string {
return query
.replace(/\\/g, '\\\\')
.replace(/%/g, '\\%')
.replace(/_/g, '\\_');
}

function cjkLikePattern(query: string): string {
return `%${escapeLikeQuery(query)}%`;
}

function cjkLikeCondition(alias: string, query: string): string {
const columns = [
'text',
'author_handle',
'author_name',
'article_title',
'article_text',
'article_site',
'links_json',
];
const perTermCondition = `(${columns.map((column) => `COALESCE(${alias}.${column}, '') LIKE ? ESCAPE '\\'`).join(' OR ')})`;
return cjkSearchTerms(query).map(() => perTermCondition).join(' AND ');
}

function cjkLikeParams(query: string): string[] {
return cjkSearchTerms(query).flatMap((term) => {
const pattern = cjkLikePattern(term);
return Array.from({ length: 7 }, () => pattern);
});
}

function cjkSearchTerms(query: string): string[] {
const terms = query
.trim()
.split(/\s+/)
.map((term) => term.trim())
.filter(Boolean);
return terms.length ? terms : [query.trim()];
}

export async function searchBookmarks(options: SearchOptions): Promise<SearchResult[]> {
const dbPath = twitterBookmarksIndexPath();
const db = await openDb(dbPath);
ensureMigrations(db);
const limit = options.limit ?? 20;
const useCjkSearch = Boolean(options.query && isCjkQuery(options.query));

try {
const conditions: string[] = [];
const params: any[] = [];

if (options.query) {
conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`);
params.push(sanitizeFtsQuery(options.query));
if (useCjkSearch) {
conditions.push(cjkLikeCondition('b', options.query));
params.push(...cjkLikeParams(options.query));
} else {
conditions.push(`b.rowid IN (SELECT rowid FROM bookmarks_fts WHERE bookmarks_fts MATCH ?)`);
params.push(sanitizeFtsQuery(options.query));
}
}
if (options.author) {
conditions.push(`b.author_handle = ? COLLATE NOCASE`);
Expand All @@ -566,13 +622,13 @@ export async function searchBookmarks(options: SearchOptions): Promise<SearchRes
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';

// If we have an FTS query, use bm25 for ranking; otherwise sort by posted_at
const orderBy = options.query
const orderBy = options.query && !useCjkSearch
? `ORDER BY bm25(bookmarks_fts, 5.0, 1.0, 1.0, 3.0) ASC`
: `ORDER BY b.posted_at DESC`;

// For FTS ranking we need to join with the FTS table for bm25
let sql: string;
if (options.query) {
if (options.query && !useCjkSearch) {
sql = `
SELECT b.id, b.url, b.text, b.author_handle, b.author_name, b.posted_at,
bm25(bookmarks_fts, 5.0, 1.0, 1.0, 3.0) as score
Expand Down
48 changes: 47 additions & 1 deletion tests/bookmarks-db.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import { mkdtemp, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { buildIndex, searchBookmarks, getStats, formatSearchResults, getBookmarkById, listBookmarks, sanitizeFtsQuery, getCategoryCounts, sampleByCategory, getClassificationProgress } from '../src/bookmarks-db.js';
import { buildIndex, searchBookmarks, getStats, formatSearchResults, getBookmarkById, listBookmarks, countBookmarks, sanitizeFtsQuery, getCategoryCounts, sampleByCategory, getClassificationProgress } from '../src/bookmarks-db.js';
import { openDb, saveDb } from '../src/db.js';
import { twitterBookmarksIndexPath } from '../src/paths.js';

Expand All @@ -13,6 +13,13 @@ const FIXTURES = [
{ id: '3', tweetId: '3', url: 'https://x.com/alice/status/3', text: 'Deep learning models need massive compute', authorHandle: 'alice', authorName: 'Alice Smith', syncedAt: '2026-03-01T00:00:00Z', postedAt: '2026-03-01T12:00:00Z', language: 'en', engagement: { likeCount: 200, repostCount: 30 }, mediaObjects: [{ type: 'photo', url: 'https://img.com/1.jpg' }], links: [], tags: [], ingestedVia: 'graphql' },
];

const CJK_FIXTURES = [
...FIXTURES,
{ id: '4', tweetId: '4', url: 'https://x.com/xiaohu/status/4', text: '一个非常狠的提示词，超级严厉的老师会一直追问。', authorHandle: 'xiaohu', authorName: '小互', syncedAt: '2026-04-01T00:00:00Z', postedAt: '2026-04-01T12:00:00Z', language: 'zh', engagement: { likeCount: 300 }, mediaObjects: [], links: [], tags: [], ingestedVia: 'graphql' },
{ id: '5', tweetId: '5', url: 'https://x.com/tools/status/5', text: 'AI 工具链需要可靠的本地搜索。', authorHandle: 'tools', authorName: '工具箱', syncedAt: '2026-05-01T00:00:00Z', postedAt: '2026-05-01T12:00:00Z', language: 'zh', engagement: { likeCount: 120 }, mediaObjects: [], links: ['https://example.com/中文搜索'], tags: [], ingestedVia: 'graphql' },
{ id: '6', tweetId: '6', url: 'https://x.com/memory/status/6', text: '这些未读内容来自 X、播客、微信、RSS 等信息流。', authorHandle: 'memory', authorName: '记忆系统', syncedAt: '2026-06-01T00:00:00Z', postedAt: '2026-06-01T12:00:00Z', language: 'zh', engagement: { likeCount: 80 }, mediaObjects: [], links: [], tags: [], ingestedVia: 'graphql' },
];

async function withIsolatedDataDir(fn: () => Promise<void>, fixtures: any[] = FIXTURES): Promise<void> {
const dir = await mkdtemp(path.join(tmpdir(), 'ft-test-'));
const jsonl = fixtures.map((r) => JSON.stringify(r)).join('\n') + '\n';
Expand Down Expand Up @@ -143,6 +150,45 @@ test('searchBookmarks: no results for unmatched query', async () => {
});
});

test('searchBookmarks: CJK query matches Chinese substrings', async () => {
await withIsolatedDataDir(async () => {
await buildIndex();
const results = await searchBookmarks({ query: '提示词', limit: 10 });
assert.equal(results.length, 1);
assert.equal(results[0]?.id, '4');
}, CJK_FIXTURES);
});

test('searchBookmarks: CJK phrase query matches Chinese text literally', async () => {
await withIsolatedDataDir(async () => {
await buildIndex();
const results = await searchBookmarks({ query: '严厉的老师', limit: 10 });
assert.equal(results.length, 1);
assert.equal(results[0]?.id, '4');
}, CJK_FIXTURES);
});

test('listBookmarks and countBookmarks: CJK query uses the same substring search path', async () => {
await withIsolatedDataDir(async () => {
await buildIndex();
const listed = await listBookmarks({ query: '工具箱', limit: 10 });
assert.equal(listed.length, 1);
assert.equal(listed[0]?.id, '5');

const count = await countBookmarks({ query: '工具箱' });
assert.equal(count, 1);
}, CJK_FIXTURES);
});

test('searchBookmarks: CJK queries split on whitespace and match terms across punctuation', async () => {
await withIsolatedDataDir(async () => {
await buildIndex();
const results = await searchBookmarks({ query: '微信 RSS', limit: 10 });
assert.equal(results.length, 1);
assert.equal(results[0]?.id, '6');
}, CJK_FIXTURES);
});

test('getStats returns correct aggregate data', async () => {
await withIsolatedDataDir(async () => {
await buildIndex();
Expand Down