Enhance search handling for Newznab and NZBHydra by incorporating ASCII title support and refining query parameter logic. This update ensures non-ASCII characters are managed effectively, improving search accuracy and compatibility.

strxno · strxno · commit eee50706068b · 2026-01-03T20:17:26.000+01:00
diff --git a/server.js b/server.js
@@ -1542,7 +1542,7 @@ async function streamHandler(req, res) {
     if (!usingCachedSearchResults) {
       const searchPlans = [];
       const seenPlans = new Set();
-      const addPlan = (planType, { tokens = [], rawQuery = null } = {}) => {
+      const addPlan = (planType, { tokens = [], rawQuery = null, asciiTitle = null } = {}) => {
         const tokenList = [...tokens];
         if (planType === 'tvsearch') {
           if (seasonToken) tokenList.push(seasonToken);
@@ -1559,6 +1559,10 @@ async function streamHandler(req, res) {
         }
         seenPlans.add(planKey);
         const planRecord = { type: planType, query, rawQuery: rawQuery ? rawQuery : null, tokens: normalizedTokens };
+        // Store asciiTitle for Newznab searches to avoid non-ASCII character issues
+        if (asciiTitle) {
+          planRecord.asciiTitle = asciiTitle;
+        }
         if (strictTextMode && planType === 'search' && rawQuery) {
           const strictPhrase = sanitizeStrictSearchPhrase(rawQuery);
           if (strictPhrase) {
@@ -1570,8 +1574,8 @@ async function streamHandler(req, res) {
         return true;
       };
 
-      // Add ID-based searches immediately (before waiting for TMDb/Cinemeta)
-      // Note: movieTitle might not be available yet, will be added later if needed
+      // Add ID-based searches (will start after title is resolved)
+      // Note: We'll add the title to these plans before executing them
       if (type === 'series' && metaIds.tvdb) {
         addPlan('tvsearch', { tokens: [`{TvdbId:${metaIds.tvdb}}`] });
       }
@@ -1584,21 +1588,6 @@ async function streamHandler(req, res) {
         addPlan(searchType, { tokens: [`{ImdbId:${metaIds.imdb}}`] });
       }
 
-      // Start ID-based searches immediately in background
-      const idSearchPromises = [];
-      const idSearchStartTs = Date.now();
-      if (searchPlans.length > 0) {
-        console.log(`${INDEXER_LOG_PREFIX} Starting ${searchPlans.length} ID-based search(es) immediately`);
-        idSearchPromises.push(...searchPlans.map((plan) => {
-          console.log(`${INDEXER_LOG_PREFIX} Dispatching early ID plan`, plan);
-          const planStartTs = Date.now();
-          return Promise.allSettled([
-            executeManagerPlanWithBackoff(plan),
-            executeNewznabPlan(plan),
-          ]).then((settled) => ({ plan, settled, startTs: planStartTs, endTs: Date.now() }));
-        }));
-      }
-
       // Now wait for TMDb to get localized titles (if applicable)
       const tmdbWaitStartTs = Date.now();
       if (tmdbMetadataPromise) {
@@ -1677,16 +1666,57 @@ async function streamHandler(req, res) {
 
       // Update ID-based plans with title if it's now available
       // This ensures NZBHydra searches include the title even when tokens are present
+      // For Newznab, prefer ASCII version if title contains non-ASCII characters
       if (movieTitle && movieTitle.trim()) {
+        const hasNonAscii = /[^\x00-\x7F]/.test(movieTitle);
+        let titleToUse = movieTitle.trim();
+        let asciiTitleToUse = null;
+        
+        // Check if we have ASCII version from TMDB
+        const tmdbTitles = metaSources.find(s => s?._tmdbTitles)?._tmdbTitles;
+        if (hasNonAscii && tmdbTitles) {
+          // Find English title or ASCII version
+          const englishTitle = tmdbTitles.find(t => t.language?.startsWith('en-'));
+          if (englishTitle) {
+            titleToUse = englishTitle.title;
+            asciiTitleToUse = englishTitle.asciiTitle || null;
+          } else {
+            // Try to find any ASCII title
+            const asciiTitleObj = tmdbTitles.find(t => t.asciiTitle && !/[^\x00-\x7F]/.test(t.asciiTitle));
+            if (asciiTitleObj) {
+              titleToUse = asciiTitleObj.asciiTitle;
+            }
+          }
+        }
+        
         searchPlans.forEach((plan) => {
           if (plan.tokens && plan.tokens.length > 0 && !plan.rawQuery) {
             // Add title to rawQuery so it's included in the search
-            plan.rawQuery = movieTitle.trim();
-            console.log(`${INDEXER_LOG_PREFIX} Updated ID plan with title: "${movieTitle}"`, { type: plan.type, tokens: plan.tokens });
+            plan.rawQuery = titleToUse;
+            if (asciiTitleToUse) {
+              plan.asciiTitle = asciiTitleToUse;
+            }
+            console.log(`${INDEXER_LOG_PREFIX} Updated ID plan with title: "${titleToUse}"`, { type: plan.type, tokens: plan.tokens, hasNonAscii });
           }
         });
       }
 
+      // Start ID-based searches now that we have the title
+      const idSearchPromises = [];
+      const idSearchStartTs = Date.now();
+      const idPlansToExecute = searchPlans.filter(p => p.tokens && p.tokens.length > 0);
+      if (idPlansToExecute.length > 0) {
+        console.log(`${INDEXER_LOG_PREFIX} Starting ${idPlansToExecute.length} ID-based search(es) with title`);
+        idSearchPromises.push(...idPlansToExecute.map((plan) => {
+          console.log(`${INDEXER_LOG_PREFIX} Dispatching ID plan`, plan);
+          const planStartTs = Date.now();
+          return Promise.allSettled([
+            executeManagerPlanWithBackoff(plan),
+            executeNewznabPlan(plan),
+          ]).then((settled) => ({ plan, settled, startTs: planStartTs, endTs: Date.now() }));
+        }));
+      }
+
       // Continue with text-based searches using TMDb titles
       const textQueryParts = [];
       let tmdbLocalizedQuery = null;
@@ -1718,9 +1748,21 @@ async function streamHandler(req, res) {
         const fallbackIdentifier = hasTmdbTitles ? null : (incomingImdbId || baseIdentifier);
         textQueryFallbackValue = (textQueryCandidate || fallbackIdentifier || '').trim();
         if (textQueryFallbackValue) {
-          const addedTextPlan = addPlan('search', { rawQuery: textQueryFallbackValue });
+          // Check if we need ASCII version for Newznab
+          const hasNonAscii = /[^\x00-\x7F]/.test(textQueryFallbackValue);
+          let asciiFallback = null;
+          if (hasNonAscii) {
+            const tmdbTitles = metaSources.find(s => s?._tmdbTitles)?._tmdbTitles;
+            if (tmdbTitles) {
+              const englishTitle = tmdbTitles.find(t => t.language?.startsWith('en-'));
+              if (englishTitle) {
+                asciiFallback = `${englishTitle.asciiTitle || englishTitle.title}${textQueryFallbackValue.replace(/^[^\x00-\x7F]+/, '').replace(/[^\x00-\x7F]+$/, '')}`.trim();
+              }
+            }
+          }
+          const addedTextPlan = addPlan('search', { rawQuery: textQueryFallbackValue, asciiTitle: asciiFallback });
           if (addedTextPlan) {
-            console.log(`${INDEXER_LOG_PREFIX} Added text search plan`, { query: textQueryFallbackValue });
+            console.log(`${INDEXER_LOG_PREFIX} Added text search plan`, { query: textQueryFallbackValue, hasNonAscii, asciiFallback: asciiFallback || 'none' });
           } else {
             console.log(`${INDEXER_LOG_PREFIX} Text search plan already present`, { query: textQueryFallbackValue });
           }
@@ -1734,28 +1776,36 @@ async function streamHandler(req, res) {
         if (tmdbTitles && tmdbTitles.length > 0 && !isSpecialRequest) {
           console.log(`[TMDB] Adding ${tmdbTitles.length} language-specific search plans`);
           tmdbTitles.forEach((titleObj) => {
-            let localizedQuery = titleObj.title;
+            // For titles with non-ASCII characters, prefer ASCII version for Newznab
+            // Store both in the plan so Newznab can choose ASCII automatically
+            const hasNonAscii = /[^\x00-\x7F]/.test(titleObj.title);
+            const queryTitle = hasNonAscii && titleObj.asciiTitle ? titleObj.asciiTitle : titleObj.title;
+            
+            let localizedQuery = queryTitle;
             if (type === 'movie' && Number.isFinite(releaseYear)) {
               localizedQuery = `${localizedQuery} ${releaseYear}`;
             } else if (type === 'series' && Number.isFinite(seasonNum) && Number.isFinite(episodeNum)) {
               localizedQuery = `${localizedQuery} S${String(seasonNum).padStart(2, '0')}E${String(episodeNum).padStart(2, '0')}`;
             }
-            const added = addPlan('search', { rawQuery: localizedQuery });
-            if (added) {
-              console.log(`${INDEXER_LOG_PREFIX} Added TMDb ${titleObj.language} search plan`, { query: localizedQuery });
-            }
-
-            // Add ASCII fallback if different
-            if (titleObj.asciiTitle && titleObj.asciiTitle !== titleObj.title) {
-              let asciiQuery = titleObj.asciiTitle;
+            
+            // Include asciiTitle in plan for Newznab to use if needed
+            const asciiQueryTitle = titleObj.asciiTitle && titleObj.asciiTitle !== titleObj.title ? titleObj.asciiTitle : null;
+            let asciiQuery = null;
+            if (asciiQueryTitle) {
+              asciiQuery = asciiQueryTitle;
               if (type === 'movie' && Number.isFinite(releaseYear)) {
                 asciiQuery = `${asciiQuery} ${releaseYear}`;
               } else if (type === 'series' && Number.isFinite(seasonNum) && Number.isFinite(episodeNum)) {
                 asciiQuery = `${asciiQuery} S${String(seasonNum).padStart(2, '0')}E${String(episodeNum).padStart(2, '0')}`;
               }
-              const addedAscii = addPlan('search', { rawQuery: asciiQuery });
-              if (addedAscii) {
-                console.log(`${INDEXER_LOG_PREFIX} Added TMDb ${titleObj.language} ASCII search plan`, { query: asciiQuery });
+            }
+            
+            const added = addPlan('search', { rawQuery: localizedQuery, asciiTitle: asciiQuery });
+            if (added) {
+              if (hasNonAscii && asciiQuery) {
+                console.log(`${INDEXER_LOG_PREFIX} Added TMDb ${titleObj.language} search plan (using ASCII for Newznab)`, { query: localizedQuery, asciiQuery });
+              } else {
+                console.log(`${INDEXER_LOG_PREFIX} Added TMDb ${titleObj.language} search plan`, { query: localizedQuery });
               }
             }
 
@@ -1981,7 +2031,10 @@ async function streamHandler(req, res) {
       }
 
       // Now execute remaining text-based search plans (exclude already-processed ID plans)
-      const remainingPlans = searchPlans.filter(p => !processedIdPlans.has(`${p.type}|${p.query}`));
+      const remainingPlans = searchPlans.filter(p => {
+        const planKey = `${p.type}|${p.query}`;
+        return !processedIdPlans.has(planKey) && (!p.tokens || p.tokens.length === 0);
+      });
       console.log(`${INDEXER_LOG_PREFIX} Executing ${remainingPlans.length} text-based search plan(s)`);
       const textSearchStartTs = Date.now();
       const planExecutions = remainingPlans.map((plan) => {
diff --git a/src/services/indexer.js b/src/services/indexer.js
@@ -161,24 +161,30 @@ function buildHydraSearchParams(plan) {
     plan.tokens.forEach((token) => applyTokenToHydraParams(token, params));
   }
 
-  // Include title in query parameter even when tokens are present
-  // NZBHydra can use both structured params (season/ep) and text query together
+  // Always include title in query parameter when available
+  // NZBHydra can use both structured params (season/ep/imdbid) and text query together
+  // This helps find results even when structured params don't match exactly
   if (plan.rawQuery) {
-    params.q = plan.rawQuery;
+    // rawQuery takes priority - it's the explicit title/text query
+    params.q = plan.rawQuery.trim();
   } else if (plan.query) {
-    // Extract title from query if it contains tokens
-    // If query is just tokens (like "{ImdbId:tt123} {Season:1} {Episode:1}"), 
-    // we need to get the title from elsewhere or use the query as-is
-    // For now, use the query which may contain the title if it was included
     const queryText = plan.query.trim();
-    // Only set q if it's not just tokens (tokens are already in structured params)
-    // Check if query looks like it contains actual text, not just token patterns
+    // Check if query is just tokens (like "{ImdbId:tt123} {Season:1} {Episode:1}")
     const isJustTokens = /^\{[^}]+\}(\s+\{[^}]+\})*$/.test(queryText);
-    if (!isJustTokens || !plan.tokens || plan.tokens.length === 0) {
+    // If it's just tokens, don't set q (we don't want to search for the literal token string)
+    // If it contains actual text, use it
+    if (!isJustTokens) {
       params.q = queryText;
     }
+    // If it's just tokens and we have structured params, we intentionally leave q empty
+    // This means the search will rely only on structured params (imdbid, season, ep)
+    // which might be why we're getting undefined results
   }
 
+  // If we have tokens but no query text, and this is a tvsearch, we should still try to include a title
+  // But we can't do that here since we don't have access to the title
+  // The title should be added to rawQuery before this function is called
+
   return params;
 }
 
diff --git a/src/services/newznab.js b/src/services/newznab.js
@@ -355,18 +355,62 @@ function applyTokenToParams(token, params) {
   }
 }
 
+/**
+ * Check if string contains non-ASCII characters
+ */
+function containsNonAscii(text) {
+  if (!text || typeof text !== 'string') return false;
+  return /[^\x00-\x7F]/.test(text);
+}
+
+/**
+ * Normalize text to ASCII (strip diacritics and non-ASCII chars)
+ * Similar to TMDB's normalizeToAscii but simpler for Newznab
+ */
+function normalizeToAscii(text) {
+  if (!text || typeof text !== 'string') return '';
+  return text
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .replace(/[^\x00-\x7F]/g, '')
+    .trim();
+}
+
 function buildSearchParams(plan) {
   const params = {
     t: mapPlanType(plan?.type),
   };
   if (Array.isArray(plan?.tokens)) {
     plan.tokens.forEach((token) => applyTokenToParams(token, params));
   }
+  
+  // For Newznab indexers, prefer ASCII-only queries to avoid false results
+  // Many indexers (like DrunkenSlug) don't handle non-ASCII characters well
+  let queryText = null;
   if (plan?.rawQuery) {
-    params.q = plan.rawQuery;
+    queryText = plan.rawQuery;
   } else if ((!plan?.tokens || plan.tokens.length === 0) && plan?.query) {
-    params.q = plan.query;
+    queryText = plan.query;
+  }
+  
+  if (queryText) {
+    // If query contains non-ASCII characters, try to use ASCII version
+    if (containsNonAscii(queryText)) {
+      // Check if plan has asciiTitle from TMDB
+      const asciiVersion = plan?.asciiTitle || normalizeToAscii(queryText);
+      if (asciiVersion && asciiVersion.trim() && asciiVersion !== queryText) {
+        console.log(`[NEWZNAB] Query contains non-ASCII characters, using ASCII version: "${asciiVersion}" (was: "${queryText}")`);
+        params.q = asciiVersion;
+      } else {
+        // If we can't get ASCII version, skip the query to avoid false results
+        console.warn(`[NEWZNAB] Query contains non-ASCII characters but no ASCII version available, skipping query: "${queryText}"`);
+        // Don't set params.q - rely on structured params (imdbid, season, ep) only
+      }
+    } else {
+      params.q = queryText;
+    }
   }
+  
   return params;
 }
 
