diff --git a/package.json b/package.json index 8e21f76..da1f750 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "axiodb", - "version": "8.33.236", + "version": "9.5.0", "description": "The Pure JavaScript Alternative to SQLite. Embedded NoSQL database for Node.js with MongoDB-style queries, zero native dependencies, built-in InMemoryCache, and web GUI. Perfect for desktop apps, CLI tools, and embedded systems. No compilation, no platform issues—pure JavaScript from npm install to production.", "main": "./lib/config/DB.js", "types": "./lib/config/DB.d.ts", diff --git a/source/Services/CRUD Operation/Reader.operation.ts b/source/Services/CRUD Operation/Reader.operation.ts index 110c8f3..a5479e7 100644 --- a/source/Services/CRUD Operation/Reader.operation.ts +++ b/source/Services/CRUD Operation/Reader.operation.ts @@ -130,11 +130,49 @@ export default class Reader { // Try index-based lookup first const indexReader = new ReadIndex(this.path); - const indexedFileNames = await indexReader.getFileFromIndex(this.baseQuery); - + let indexedFileNames: string[] = []; + + // Check if query can use index optimization + const queryKeys = Object.keys(this.baseQuery); + if (queryKeys.length === 1) { + const fieldName = queryKeys[0]; + const fieldValue = this.baseQuery[fieldName]; + + if (typeof fieldValue === 'object' && fieldValue !== null) { + // OPTIMIZED: Use $in-aware index lookup (O(K) vs O(N)) + if ('$in' in fieldValue) { + indexedFileNames = await indexReader.getFilesForInOperator(fieldName, fieldValue.$in); + } + // OPTIMIZED: Use prefix index lookup for regex patterns like /^prefix/ + else if ('$regex' in fieldValue) { + const prefixInfo = this.detectPrefixPattern(fieldValue.$regex, fieldValue.$options); + if (prefixInfo.isPrefix && prefixInfo.prefix) { + indexedFileNames = await indexReader.getFilesForPrefixQuery( + fieldName, + prefixInfo.prefix, + prefixInfo.caseInsensitive + ); + } else { + // Non-prefix regex - use standard lookup (will likely fall back to full scan) + indexedFileNames = await indexReader.getFileFromIndex(this.baseQuery); + } + } + // Other operators - use standard lookup + else { + indexedFileNames = await indexReader.getFileFromIndex(this.baseQuery); + } + } else { + // Standard exact match + indexedFileNames = await indexReader.getFileFromIndex(this.baseQuery); + } + } else { + // Multiple fields or no fields - use standard index lookup + indexedFileNames = await indexReader.getFileFromIndex(this.baseQuery); + } + let ReadResponse; let usedIndex = false; - + if (indexedFileNames && indexedFileNames.length > 0) { // Index hit - load only indexed files (much faster) ReadResponse = await this.LoadAllBufferRawData(indexedFileNames); @@ -179,12 +217,47 @@ export default class Reader { private isExactIndexMatch(): boolean { const queryKeys = Object.keys(this.baseQuery); if (queryKeys.length !== 1) return false; - + const value = this.baseQuery[queryKeys[0]]; // Exact match if value is primitive (not an operator object) return typeof value !== 'object' || value === null; } + /** + * Detects if a regex pattern is a simple prefix match (e.g., /^John/, /^admin@/) + * and extracts the prefix for index optimization + * + * @param regex - The regex pattern (RegExp object or string) + * @param options - Optional regex flags (e.g., 'i' for case-insensitive) + * @returns Object with isPrefix flag, prefix string, and case-insensitive flag + * + * @example + * detectPrefixPattern(/^John/, 'i') // { isPrefix: true, prefix: 'John', caseInsensitive: true } + * detectPrefixPattern(/John/) // { isPrefix: false } + * detectPrefixPattern(/^test[0-9]+/) // { isPrefix: false } - complex pattern + */ + private detectPrefixPattern( + regex: RegExp | string, + options?: string + ): { isPrefix: boolean; prefix?: string; caseInsensitive: boolean } { + const regexStr = regex instanceof RegExp ? regex.source : String(regex); + const flags = regex instanceof RegExp ? regex.flags : (options || ''); + + // Match simple prefix patterns: ^abc, ^test, ^John (alphanumeric, underscore, dash, @, .) + // Excludes complex patterns with [], *, +, {}, etc. + const prefixMatch = regexStr.match(/^\^([a-zA-Z0-9_\-@.]+)$/); + + if (prefixMatch) { + return { + isPrefix: true, + prefix: prefixMatch[1], + caseInsensitive: flags.includes('i') + }; + } + + return { isPrefix: false, caseInsensitive: false }; + } + /** * Applies sorting if needed and returns data with skip/limit */ diff --git a/source/Services/Index/ReadIndex.service.ts b/source/Services/Index/ReadIndex.service.ts index b3b66d1..b6b75c3 100644 --- a/source/Services/Index/ReadIndex.service.ts +++ b/source/Services/Index/ReadIndex.service.ts @@ -69,6 +69,86 @@ export class ReadIndex extends IndexManager { } } + /** + * Retrieve file paths from an index for documents matching any value in the $in array. + * + * OPTIMIZED: Uses index lookups for each value in the $in array, unions the results. + * This is significantly faster than full collection scan for indexed fields. + * + * @param fieldName - The field name to query (must have an index) + * @param values - Array of values to match (from $in operator) + * + * @returns Promise resolving to array of unique file paths matching any value + * + * @remarks + * - Uses Set for automatic deduplication of file paths + * - Returns empty array if field has no index + * - O(K) lookups where K = values.length (much faster than O(N) full scan) + * + * @example + * // For query: { category: { $in: ['Electronics', 'Books'] } } + * const files = await readIndex.getFilesForInOperator('category', ['Electronics', 'Books']); + */ + public async getFilesForInOperator(fieldName: string, values: any[]): Promise { + const indexData = await this.indexCache.getIndex(fieldName); + if (!indexData) return []; + + const fileSet = new Set(); + for (const value of values) { + const files = indexData.indexEntries[value]; + if (files) { + files.forEach(f => fileSet.add(f)); + } + } + return Array.from(fileSet); + } + + /** + * Retrieve file paths from an index for documents where field value starts with a prefix. + * + * OPTIMIZED: Uses index to filter values by prefix, avoiding full collection scan. + * Works with hash-based indexes by filtering index keys. + * + * @param fieldName - The field name to query (must have an index) + * @param prefix - The prefix string to match + * @param caseInsensitive - Whether to perform case-insensitive matching (default: false) + * + * @returns Promise resolving to array of unique file paths where field starts with prefix + * + * @remarks + * - Filters index keys for prefix matches (O(K) where K = index key count) + * - Much faster than full collection scan for prefix patterns + * - Falls back to empty array if field has no index + * - Best used for regex patterns like /^John/ or /^admin@/ + * + * @example + * // For query: { name: { $regex: /^John/i } } + * const files = await readIndex.getFilesForPrefixQuery('name', 'John', true); + */ + public async getFilesForPrefixQuery( + fieldName: string, + prefix: string, + caseInsensitive: boolean = false + ): Promise { + const indexData = await this.indexCache.getIndex(fieldName); + if (!indexData) return []; + + const normalizedPrefix = caseInsensitive ? prefix.toLowerCase() : prefix; + const fileSet = new Set(); + + // Iterate through index keys and find matches + // For hash-based indexes, this is O(K) where K = number of unique values + for (const [value, files] of Object.entries(indexData.indexEntries)) { + const normalizedValue = caseInsensitive ? value.toLowerCase() : value; + + if (normalizedValue.startsWith(normalizedPrefix)) { + files.forEach(f => fileSet.add(f)); + } + } + + return Array.from(fileSet); + } + /** * Finds index metadata entries that correspond to properties present on the provided document. * diff --git a/source/utility/Searcher.utils.ts b/source/utility/Searcher.utils.ts index 7546314..551bf58 100644 --- a/source/utility/Searcher.utils.ts +++ b/source/utility/Searcher.utils.ts @@ -157,7 +157,7 @@ export default class Searcher { const effectiveLimit = findOne ? 1 : limit; // For small datasets, findOne, or when limit is small - use optimized linear search - if (this.data.length < 5000 || findOne || (effectiveLimit && effectiveLimit < 100)) { + if (findOne || (effectiveLimit && effectiveLimit < 1000) || this.data.length < 10000) { const result: any[] = []; for (let i = 0; i < this.data.length; i++) { const rawItem = this.data[i]; @@ -250,6 +250,16 @@ export default class Searcher { return andMatch && restMatch; } + // Handle root-level $nor (negated OR - none of the conditions should match) + if ("$nor" in query && Array.isArray(query.$nor)) { + const { $nor, ...rest } = query; + const norMatch = !$nor.some((sub) => this.matchesQuery(item, sub)); + const restMatch = Object.keys(rest).length + ? this.matchesQuery(item, rest) + : true; + return norMatch && restMatch; + } + // Two-pointer optimized query matching const queryKeys = Object.keys(query); const queryLength = queryKeys.length; @@ -298,6 +308,70 @@ export default class Searcher { continue; } + // $exists - Check if field exists in document + if ("$exists" in queryValue) { + const shouldExist = queryValue["$exists"]; + const fieldExists = itemValue !== undefined && itemValue !== null; + if (shouldExist && !fieldExists) return false; + if (!shouldExist && fieldExists) return false; + continue; + } + + // $elemMatch - Match array elements with nested conditions + if ("$elemMatch" in queryValue) { + if (!Array.isArray(itemValue)) return false; + + const elemQuery = queryValue["$elemMatch"]; + const hasMatch = itemValue.some(elem => { + return this.matchesQuery(elem, elemQuery, false); + }); + + if (!hasMatch) return false; + continue; + } + + // $not - Negation of query condition + if ("$not" in queryValue) { + const negatedQuery = queryValue["$not"]; + const tempDoc = { [key]: itemValue }; + const tempQuery = { [key]: negatedQuery }; + + if (this.matchesQuery(isUpdated ? { data: tempDoc } : tempDoc, tempQuery, isUpdated)) { + return false; + } + continue; + } + + // $type - Check value type + if ("$type" in queryValue) { + const expectedType = queryValue["$type"]; + let actualType = itemValue === null ? 'null' + : Array.isArray(itemValue) ? 'array' + : typeof itemValue; + + if (actualType !== expectedType) return false; + continue; + } + + // $size - Check array length + if ("$size" in queryValue) { + if (!Array.isArray(itemValue)) return false; + if (itemValue.length !== queryValue["$size"]) return false; + continue; + } + + // $all - Array must contain all specified values + if ("$all" in queryValue && Array.isArray(queryValue["$all"])) { + if (!Array.isArray(itemValue)) return false; + + const requiredValues = queryValue["$all"]; + const itemSet = new Set(itemValue); + const hasAll = requiredValues.every(val => itemSet.has(val)); + + if (!hasAll) return false; + continue; + } + if ("$eq" in queryValue) { if (itemValue !== queryValue["$eq"]) return false; continue;