Merge pull request #25 from SnappyLab/2-add-the-capability-to-search-for-documents-metadata

2 add the capability to search for documents metadata

Author: ChristopheBeke

.github/workflows/docbinder-oss.yml

@@ -5,10 +5,16 @@ on:
     branches:
       - main
       - dev
+    paths-ignore:
+      - "docs/**"
+      - "mkdocs.yml"
   pull_request:
     branches:
       - main
       - dev
+    paths-ignore:
+      - "docs/**"
+      - "mkdocs.yml"
 jobs:
   test:
     runs-on: ubuntu-latest

.gitignore

@@ -77,3 +77,7 @@ ENV/
 # Credentials
 gcp_credentials.json
 *_token.json
+
+# Test files
+search_results.csv
+search_results.json

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.7.16
+    hooks:
+      - id: uv-export
+      - id: uv-lock
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.12.1
+    hooks:
+    # Run the linter.
+    - id: ruff-check
+      types_or: [ python, pyi ]
+      args: [ --select, I, --fix, --select=E501 ]
+    # Run the formatter.
+    - id: ruff-format
+      types_or: [ python, pyi ]

CONTRIBUTING.md

@@ -56,4 +56,26 @@ All dependencies are tracked in `pyproject.toml`. Use `uv` commands to keep it u
 ---
 
 **Note:**  
-Always use `uv` commands to manage dependencies and environments to keep `pyproject.toml` in sync.
+Always use `uv` commands to manage dependencies and environments to keep `pyproject.toml` in sync.
+
+## Code Style and Linting
+
+This project uses [Black](https://black.readthedocs.io/en/stable/) for code formatting and [Ruff](https://docs.astral.sh/ruff/) for linting. All code should be formatted and linted before committing.
+
+- Run the following before committing code:
+
+```zsh
+uv run black .
+uv run ruff check .
+```
+
+- To automatically format and lint code on every commit, install pre-commit hooks:
+
+```zsh
+uv pip install pre-commit
+pre-commit install
+```
+
+This will ensure Black and Ruff are run on staged files before each commit.
+
+Configuration for Black and Ruff is in `pyproject.toml`. This enforces consistent quotes, spacing, and other style rules for all contributors.

docs/CONTRIBUTING.md

@@ -56,4 +56,26 @@ All dependencies are tracked in `pyproject.toml`. Use `uv` commands to keep it u
 ---
 
 **Note:**  
-Always use `uv` commands to manage dependencies and environments to keep `pyproject.toml` in sync.
+Always use `uv` commands to manage dependencies and environments to keep `pyproject.toml` in sync.
+
+## Code Style and Linting
+
+This project uses [Black](https://black.readthedocs.io/en/stable/) for code formatting and [Ruff](https://docs.astral.sh/ruff/) for linting. All code should be formatted and linted before committing.
+
+- Run the following before committing code:
+
+```zsh
+uv run black .
+uv run ruff check .
+```
+
+- To automatically format and lint code on every commit, install pre-commit hooks:
+
+```zsh
+uv pip install pre-commit
+pre-commit install
+```
+
+This will ensure Black and Ruff are run on staged files before each commit.
+
+Configuration for Black and Ruff is in `pyproject.toml`. This enforces consistent quotes, spacing, and other style rules for all contributors.

docs/tool/providers/custom_provider.md

@@ -0,0 +1,119 @@
+# How to Add a New Provider
+
+This guide explains how to integrate a new storage provider (e.g., DropBox, OneDrive) into DocBinder-OSS. The process involves creating configuration and client classes, registering the provider, and ensuring compatibility with the system’s models and interfaces.
+
+---
+
+## 1. Create a Service Configuration Class
+
+Each provider must define a configuration class that inherits from [`ServiceConfig`](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/services/base_class.py):
+
+```python
+# filepath: src/docbinder_oss/services/my_provider/my_provider_service_config.py
+from docbinder_oss.services.base_class import ServiceConfig
+
+class MyProviderServiceConfig(ServiceConfig):
+    type: str = "my_provider"
+    name: str
+    # Add any other provider-specific fields here
+    api_key: str
+```
+
+- `type` must be unique and match the provider’s identifier.
+- `name` is a user-defined label for this provider instance.
+
+---
+
+## 2. Implement the Storage Client
+
+Create a client class that inherits from [`BaseStorageClient`](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/services/base_class.py) and implements all abstract methods:
+
+```python
+# filepath: src/docbinder_oss/services/my_provider/my_provider_client.py
+from typing import Optional, List
+from docbinder_oss.services.base_class import BaseStorageClient
+from docbinder_oss.core.schema import File, Permission
+from .my_provider_service_config import MyProviderServiceConfig
+
+class MyProviderClient(BaseStorageClient):
+    def __init__(self, config: MyProviderServiceConfig):
+        self.config = config
+        # Initialize SDK/client here
+
+    def test_connection(self) -> bool:
+        # Implement connection test
+        pass
+
+    def list_files(self, folder_id: Optional[str] = None) -> List[File]:
+        # Implement file listing
+        pass
+
+    def get_file_metadata(self, item_id: str) -> File:
+        # Implement metadata retrieval
+        pass
+
+    def get_permissions(self, item_id: str) -> List[Permission]:
+        # Implement permissions retrieval
+        pass
+```
+
+- Use the shared models [`File`](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/core/schemas.py), [`Permission`](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/core/schemas.py), etc., for return types.
+
+---
+
+## 3. Register the Provider
+
+Add an `__init__.py` in your provider’s folder with a `register()` function:
+
+```python
+# filepath: src/docbinder_oss/services/my_provider/__init__.py
+from .my_provider_client import MyProviderClient
+from .my_provider_service_config import MyProviderServiceConfig
+
+def register():
+    return {
+        "display_name": "my_provider",
+        "config_class": MyProviderServiceConfig,
+        "client_class": MyProviderClient,
+    }
+```
+
+---
+
+## 4. Ensure Discovery
+
+The system will automatically discover your provider if it’s in the `src/docbinder_oss/services/` directory and contains a `register()` function in `__init__.py`.
+
+---
+
+## 5. Update the Config File
+
+Add your provider’s configuration to `~/.config/docbinder/config.yaml`:
+
+```yaml
+providers:
+  - type: my_provider
+    name: my_instance
+    # Add other required fields
+    api_key: <your-api-key>
+```
+
+---
+
+## 6. Test Your Provider
+
+- Run the application and ensure your provider appears and works as expected.
+- The config loader will validate your config using your `ServiceConfig` subclass.
+
+---
+
+## Reference
+
+- [src/docbinder_oss/services/base_class.py](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/services/base_class.py)
+- [src/docbinder_oss/core/schemas.py](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/core/schemas.py)
+- [src/docbinder_oss/services/google_drive/](https://github.com/SnappyLab/DocBinder-OSS/tree/main/src/docbinder_oss/services/google_drive/) (example implementation)
+- [src/docbinder_oss/services/__init__.py](https://github.com/SnappyLab/DocBinder-OSS/blob/main/src/docbinder_oss/services/__init__.py)
+
+---
+
+**Tip:** Use the Google Drive as a template for your implementation. Make sure to follow the abstract method signatures and use the shared models for compatibility.

docs/tool/providers/google_drive.md

@@ -0,0 +1,68 @@
+# Google Drive Configuration Setup
+
+This guide will help you configure Google Drive as a provider for DocBinder.
+
+## Prerequisites
+
+- A Google account
+- Access to [Google Cloud Console](https://console.cloud.google.com/)
+- DocBinder installed
+
+## Step 1: Create a Google Cloud Project
+
+1. Go to the [Google Cloud Console](https://console.cloud.google.com/).
+2. Click on **Select a project** and then **New Project**.
+3. Enter a project name and click **Create**.
+
+## Step 2: Enable Google Drive API
+
+1. In your project dashboard, navigate to **APIs & Services > Library**.
+2. Search for **Google Drive API**.
+3. Click **Enable**.
+
+## Step 3: Create OAuth 2.0 Credentials
+
+1. Go to **APIs & Services > Credentials**.
+2. Click **+ CREATE CREDENTIALS** and select **OAuth client ID**.
+3. Configure the consent screen if prompted.
+4. Choose **Desktop app** or **Web application** as the application type.
+5. Enter a name and click **Create**.
+6. Download the `credentials.json` file.
+
+## Step 4: Configure DocBinder
+
+1. Place your downloaded credentials file somewhere accessible (e.g., ~/gcp_credentials.json).
+2. The application will generate a token file (e.g., ~/gcp_token.json) after the first authentication.
+   
+## Step 5: Edit the Config File
+
+Create the config file, and add a provider entry for Google Drive:
+```yaml
+providers:
+  - type: google_drive
+    name: my_gdrive
+    gcp_credentials_json: ./gcp_credentials.json
+    gcp_token_json: ./gcp_token.json
+```
+
+* type: Must be google_drive.
+* name: A unique name for this provider.
+* gcp_credentials_json: Absolute/relative path to your Google Cloud credentials file.
+* gcp_token_json: Absolute/relative path where the token will be stored/generated.
+
+## Step 6: Authenticate and Test
+
+1. Run DocBinder with the Google Drive provider enabled.
+2. On first run, follow the authentication prompt to grant access.
+3. Verify that DocBinder can access your Google Drive files.
+
+## Troubleshooting
+
+- Ensure your credentials file is in the correct location.
+- Check that the Google Drive API is enabled for your project.
+- Review the [Google API Console](https://console.developers.google.com/) for error messages.
+
+## References
+
+- [Google Drive API Documentation](https://developers.google.com/drive)
+- [DocBinder OSS - GitHub](https://github.com/SnappyLab/DocBinder-OSS)

mkdocs.yml

@@ -10,6 +10,9 @@ nav:
       - Commands:
           - Main CLI: commands/main.md
           - Provider: commands/provider.md
+  - Providers:
+      - Google Drive: tool/providers/google_drive.md
+      - Custom Provider: tool/providers/custom_provider.md
   - Contributing: CONTRIBUTING.md
   - Code of Conduct: CODE_OF_CONDUCT.md
   - Security: SECURITY.md

provider_setup_example.yml

@@ -0,0 +1,4 @@
+providers:
+  - type: google_drive
+    name: my_google_drive
+    gcp_credentials_json: gcp_credentials.json

pyproject.toml

@@ -32,8 +32,10 @@ include = ["src/docbinder_oss/**"]
 
 [dependency-groups]
 dev = [
+    "black>=25.1.0",
     "mkdocs>=1.6.1",
     "mkdocs-material>=9.6.14",
+    "pre-commit>=4.2.0",
     "pytest>=8.4.0",
     "tox>=4.26.0",
     "tox-uv>=1.26.0",
@@ -47,8 +49,7 @@ testpaths = [
 ]
 
 [tool.ruff]
-# Set the maximum line length to 100.
-line-length = 100
+line-length = 125
 
 [tool.ruff.lint]
 # Add the `line-too-long` rule to the enforced rule set. By default, Ruff omits rules that