[bug] Server console throws http status errors when running Web search (MacOS, MiniMax model) (#42)

* Worker loop error … httpx.ReadError.ReadError (connection dropped mid-response) not caught in URL skill -escaped to worker loop

* fix(url-skill): robust SSL and network error handling with certifi CA bundle

- Use certifi CA bundle via ssl.create_default_context to fix macOS python.org SSL failures
- Catch httpx.ConnectError in URL skill to gracefully skip SSL errors (return empty content)
  and re-raise non-SSL ConnectErrors for orchestrator retry handling
- Add httpx.ConnectError and httpx.ReadError to orchestrator warning-level retry handler
  so they are retried with backoff instead of triggering ERROR-level tracebacks
- Add certifi>=2024.0 to project dependencies
- Add test coverage: SSL graceful skip, ConnectTimeout propagation, ReadError propagation

* feat(config): expose fetch_timeout_seconds for URL skill in [ingest] config

Default remains 30s. Users can override via .synthadoc/config.toml:

  [ingest]
  fetch_timeout_seconds = 60

Threaded through IngestAgent → SkillAgent → UrlSkill. SkillAgent gains
a skill_kwargs dict for passing constructor args to named skills.

* docs: add fetch_timeout_seconds to configuration reference and quick-start guide

* Add OpenAI to the supported list in README.

Author: paulmchen

README.md

@@ -132,7 +132,7 @@ As the wiki accumulates pages the `index.md` table of contents, domain scope (`p
 | Offline browsable artifact   | **Yes**                                             | No          | No         | No        |
 | Multi-wiki isolation         | **Yes**                                             | No          | No         | No        |
 | Web search → wiki pages     | **Yes**                                             | No          | No         | No        |
-| Multiple LLMs support       | **Yes** (MiniMax, Gemini, Groq, Anthropic, Ollama) | No          | No         | No        |
+| Multiple LLMs support       | **Yes** (MiniMax, Gemini, Groq, Anthropic, OpenAI, Ollama) | No          | No         | No        |
 | Auto wiki overview page      | **Yes**                                             | No          | No         | No        |
 | Resumable job queue + retry  | **Yes**                                             | No          | No         | No        |
 | Query decomposition          | **Yes** (parallel sub-queries)                      | No          | No         | No        |

docs/design.md

@@ -854,9 +854,10 @@ hard_gate_usd                     = 2.00
 auto_resolve_confidence_threshold = 0.85
 
 [ingest]
-max_pages_per_ingest = 15
-chunk_size           = 1500
-chunk_overlap        = 150
+max_pages_per_ingest  = 15
+chunk_size            = 1500
+chunk_overlap         = 150
+fetch_timeout_seconds = 30   # seconds to wait for a URL response before retrying
 
 [logs]
 level        = "INFO"
@@ -900,6 +901,7 @@ cron = "0 3 * * 0"   # every Sunday at 03:00
 | `ingest.max_pages_per_ingest` | int | `15` | Max pages one ingest may update |
 | `ingest.chunk_size` | int | `1500` | Text chunk size (characters) |
 | `ingest.chunk_overlap` | int | `150` | Overlap between chunks |
+| `ingest.fetch_timeout_seconds` | int | `30` | Seconds to wait for a URL response before retrying |
 | `logs.level` | str | `"INFO"` | Console log level |
 | `logs.max_file_mb` | int | `5` | Rotate `synthadoc.log` at this size |
 | `logs.backup_count` | int | `5` | Rotated files to keep |

docs/user-quick-start-guide.md

@@ -976,6 +976,9 @@ port = 7071          # required if running more than one wiki simultaneously
 soft_warn_usd = 0.50
 hard_gate_usd = 2.00
 
+[ingest]
+fetch_timeout_seconds = 60   # increase if slow sites time out during web search
+
 [web_search]
 provider    = "tavily"
 max_results = 20

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "anthropic>=0.30",
     "openai>=1.40",
     "httpx>=0.27",
+    "certifi>=2024.0",
     "beautifulsoup4>=4.12",
     "pypdf>=4.0",
     "pdfminer.six>=20221105",

synthadoc/agents/ingest_agent.py

@@ -164,7 +164,8 @@ class IngestAgent:
     def __init__(self, provider: LLMProvider, store: WikiStorage, search: HybridSearch,
                  log_writer: LogWriter, audit_db: AuditDB, cache: CacheManager,
                  max_pages: int = 15, wiki_root: Optional[Path] = None,
-                 cache_version: str = CACHE_VERSION) -> None:
+                 cache_version: str = CACHE_VERSION,
+                 fetch_timeout: int = 30) -> None:
         self._provider = provider
         self._store = store
         self._search = search
@@ -174,7 +175,7 @@ def __init__(self, provider: LLMProvider, store: WikiStorage, search: HybridSear
         self._max_pages = max_pages
         self._wiki_root = Path(wiki_root) if wiki_root is not None else None
         self._cache_version = cache_version
-        self._skill_agent = SkillAgent()
+        self._skill_agent = SkillAgent(skill_kwargs={"url": {"fetch_timeout": fetch_timeout}})
         self._purpose = self._load_purpose()
 
     async def _analyse(self, text: str, bust_cache: bool = False) -> dict:

synthadoc/agents/skill_agent.py

@@ -84,9 +84,11 @@ def __init__(
         self,
         wiki_root: Optional[Path] = None,
         extra_dirs: Optional[list[Path]] = None,
+        skill_kwargs: Optional[dict[str, dict]] = None,
     ) -> None:
         self._registry: dict[str, SkillMeta] = {}
         self._loaded: dict[str, type[BaseSkill]] = {}
+        self._skill_kwargs: dict[str, dict] = skill_kwargs or {}
         self._build_registry(wiki_root, extra_dirs or [])
 
     def _build_registry(self, wiki_root: Optional[Path], extra_dirs: list[Path]) -> None:
@@ -159,7 +161,8 @@ def get_skill(self, name: str) -> BaseSkill:
             self._check_requires(meta)
             cls = _import_class(meta.skill_dir / meta.entry_script, meta.entry_class)
             self._loaded[name] = cls
-        instance = self._loaded[name]()
+        kwargs = self._skill_kwargs.get(name, {})
+        instance = self._loaded[name](**kwargs)
         instance.skill_dir = self._registry[name].skill_dir
         return instance
 

synthadoc/config.py

@@ -75,6 +75,7 @@ class IngestConfig:
     max_pages_per_ingest: int = 15
     chunk_size: int = 1500
     chunk_overlap: int = 150
+    fetch_timeout_seconds: int = 30
 
 
 @dataclass
@@ -255,6 +256,7 @@ def _raw_to_config(raw: dict, source_has_agents: bool) -> Config:
         max_pages_per_ingest=ig.get("max_pages_per_ingest", 15),
         chunk_size=ig.get("chunk_size", 1500),
         chunk_overlap=ig.get("chunk_overlap", 150),
+        fetch_timeout_seconds=ig.get("fetch_timeout_seconds", 30),
     )
 
     # --- query ---

synthadoc/core/orchestrator.py

@@ -136,6 +136,7 @@ async def _run_ingest(self, job_id: str, source: str, auto_confirm: bool,
                 log_writer=self._log, audit_db=self._audit,
                 cache=self._cache, max_pages=self._cfg.ingest.max_pages_per_ingest,
                 cache_version=self._cfg.cache.version,
+                fetch_timeout=self._cfg.ingest.fetch_timeout_seconds,
             )
             result = await agent.ingest(source, force=force, bust_cache=force)
             _agent_cfg = self._cfg.agents.resolve("ingest")
@@ -206,12 +207,14 @@ async def _run_ingest(self, job_id: str, source: str, auto_confirm: bool,
             elif isinstance(e, DomainBlockedException):
                 await self._auto_block_domain(e)
                 await self._queue.skip(job_id, str(e))
-            elif isinstance(e, (httpx.ReadTimeout, httpx.ConnectTimeout, httpx.PoolTimeout)):
-                # Transient network timeout — retry with backoff, no traceback.
+            elif isinstance(e, (httpx.ReadTimeout, httpx.ConnectTimeout, httpx.PoolTimeout,
+                                   httpx.ConnectError, httpx.ReadError)):
+                # Transient network error (timeout, connection refused, dropped) — retry with backoff.
                 logging.getLogger(__name__).warning(
-                    "URL fetch timed out for job %s (%s) — will retry", job_id, source
+                    "URL fetch failed for job %s (%s: %s) — will retry", job_id, source,
+                    type(e).__name__
                 )
-                await self._queue.fail(job_id, f"ReadTimeout: {source}")
+                await self._queue.fail(job_id, f"{type(e).__name__}: {source}")
             elif isinstance(e, httpx.HTTPStatusError):
                 status = e.response.status_code
                 if 400 <= status < 500:

synthadoc/integration/http_server.py

@@ -270,6 +270,7 @@ async def analyse_source(req: AnalyseRequest):
             cache=orch._cache, max_pages=orch._cfg.ingest.max_pages_per_ingest,
             wiki_root=orch._root,
             cache_version=orch._cfg.cache.version,
+            fetch_timeout=orch._cfg.ingest.fetch_timeout_seconds,
         )
         skill = SkillAgent()
         extracted = await skill.extract(req.source)

synthadoc/skills/url/scripts/main.py

@@ -1,12 +1,14 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # Copyright (C) 2026 Paul Chen / axoviq.com
+import logging
 import tempfile
 import httpx
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse
 from synthadoc.skills.base import BaseSkill, ExtractedContent, SkillMeta
 from synthadoc.errors import DomainBlockedException
 
+logger = logging.getLogger(__name__)
 
 _HEADERS = {
     "User-Agent": (
@@ -19,25 +21,49 @@
 # HTTP status codes that indicate bot/access blocking (not transient errors)
 _BLOCKED_STATUSES = {403, 401, 429}
 
+# macOS Python (python.org installer) doesn't use the system keychain.
+# certifi ships its own CA bundle that covers the vast majority of public sites.
+import ssl as _ssl
+try:
+    import certifi as _certifi
+    _SSL_CONTEXT = _ssl.create_default_context(cafile=_certifi.where())
+except ImportError:
+    _SSL_CONTEXT = _ssl.create_default_context()  # fall back to system certs
+
 
 class UrlSkill(BaseSkill):
     meta = SkillMeta(name="url", description="Fetch and extract text from web URLs",
                      extensions=["https://", "http://"])
 
+    def __init__(self, fetch_timeout: int = 30) -> None:
+        super().__init__()
+        self._fetch_timeout = fetch_timeout
+
     async def extract(self, source: str) -> ExtractedContent:
-        async with httpx.AsyncClient(follow_redirects=True, timeout=30, headers=_HEADERS) as client:
-            resp = await client.get(source)
-            if resp.status_code in _BLOCKED_STATUSES:
-                domain = urlparse(source).hostname or source
-                raise DomainBlockedException(
-                    domain=domain, url=source, status_code=resp.status_code
-                )
-            resp.raise_for_status()
-            content_type = resp.headers.get("content-type", "")
-            is_pdf = "application/pdf" in content_type or source.lower().endswith(".pdf")
-            if is_pdf:
-                return self._extract_pdf_response(resp.content, source)
-            html = resp.text
+        try:
+            async with httpx.AsyncClient(
+                follow_redirects=True, timeout=self._fetch_timeout, headers=_HEADERS, verify=_SSL_CONTEXT
+            ) as client:
+                resp = await client.get(source)
+        except httpx.ConnectError as exc:
+            err_str = str(exc)
+            if "CERTIFICATE_VERIFY_FAILED" in err_str or "SSL" in err_str.upper():
+                # SSL errors won't resolve on retry — skip gracefully
+                logger.warning("SSL verification failed for %s — skipping", source)
+                return ExtractedContent(text="", source_path=source,
+                                        metadata={"url": source, "ssl_error": True})
+            raise  # non-SSL ConnectError — let orchestrator handle (retry with backoff)
+        if resp.status_code in _BLOCKED_STATUSES:
+            domain = urlparse(source).hostname or source
+            raise DomainBlockedException(
+                domain=domain, url=source, status_code=resp.status_code
+            )
+        resp.raise_for_status()
+        content_type = resp.headers.get("content-type", "")
+        is_pdf = "application/pdf" in content_type or source.lower().endswith(".pdf")
+        if is_pdf:
+            return self._extract_pdf_response(resp.content, source)
+        html = resp.text
 
         soup = BeautifulSoup(html, "html.parser")
         for tag in soup(["script", "style", "nav", "footer"]):
@@ -53,12 +79,10 @@ def _extract_pdf_response(self, content: bytes, source: str) -> ExtractedContent
         an empty ExtractedContent is returned so the job completes as 'skipped'
         rather than dying after 3 retries.
         """
-        import logging
         import os
         import pypdf
 
         logging.getLogger("pypdf").setLevel(logging.ERROR)
-        logger = logging.getLogger(__name__)
 
         with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
             tmp.write(content)