Upgrade Streamlit to 1.49 (#278)

* update streamlit

* fix md5 error

* fix sidebar updates

* remove workspace selector if offline

* update workspace via query param

* fix duplicate component bug

* fix log refresh

* stream output in workflows

* fix workflow display

Author: t0mdavid-m

content/digest.py

@@ -315,7 +315,4 @@ def main():
                 st.error("Please check your input and try again. If the problem persists, try with a simpler enzyme like Trypsin.")
 
 
-if __name__ == "__main__":
-    
-    main()
 main()

requirements.txt

@@ -111,7 +111,7 @@ six==1.17.0
     # via python-dateutil
 smmap==5.0.2
     # via gitdb
-streamlit==1.43.0
+streamlit==1.49.0
     # via
     #   src (pyproject.toml)
     #   streamlit-js-eval

src/common/captcha_.py

@@ -1,14 +1,23 @@
-from pathlib import Path
 import streamlit as st
 import streamlit.components.v1 as st_components
-from streamlit.source_util import page_icon_and_name, calc_md5, get_pages, _on_pages_changed
 
+from streamlit.source_util import page_icon_and_name
 from captcha.image import ImageCaptcha
+from pathlib import Path
 
+import hashlib
 import random
 import string
 import os
 
+def calc_md5(string : str):
+    return hashlib.md5(string.encode()).hexdigest() 
+
+def get_pages():
+    return st.runtime.get_pages()
+
+def set_pages(pages : dict):
+    st.runtime.set_pages(pages)
 
 def delete_all_pages(main_script_path_str: str) -> None:
     """
@@ -22,7 +31,7 @@ def delete_all_pages(main_script_path_str: str) -> None:
 
     """
     # Get all pages from the app's configuration
-    current_pages = get_pages(main_script_path_str)
+    current_pages = get_pages()
 
     # Create a list to store keys pages to delete
     keys_to_delete = []
@@ -37,7 +46,7 @@ def delete_all_pages(main_script_path_str: str) -> None:
         del current_pages[key]
 
     # Refresh the pages configuration
-    _on_pages_changed.send()
+    set_pages(current_pages)
 
 
 def delete_page(main_script_path_str: str, page_name: str) -> None:
@@ -52,15 +61,15 @@ def delete_page(main_script_path_str: str, page_name: str) -> None:
         None
     """
     # Get all pages
-    current_pages = get_pages(main_script_path_str)
+    current_pages = get_pages()
 
     # Iterate over all pages and delete the desired page if found
     for key, value in current_pages.items():
         if value["page_name"] == page_name:
             del current_pages[key]
 
     # Refresh the pages configuration
-    _on_pages_changed.send()
+    set_pages(current_pages)
 
 
 def restore_all_pages(main_script_path_str: str) -> None:
@@ -74,7 +83,7 @@ def restore_all_pages(main_script_path_str: str) -> None:
         None
     """
     # Get all pages
-    pages = get_pages(main_script_path_str)
+    pages = get_pages()
 
     # Obtain the path to the main script
     main_script_path = Path(main_script_path_str)
@@ -126,7 +135,7 @@ def restore_all_pages(main_script_path_str: str) -> None:
         }
 
     # Refresh the page configuration
-    _on_pages_changed.send()
+    set_pages(pages)
 
 
 def add_page(main_script_path_str: str, page_name: str) -> None:
@@ -141,7 +150,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None:
         None
     """
     # Get all pages
-    pages = get_pages(main_script_path_str)
+    pages = get_pages()
 
     # Obtain the path to the main script
     main_script_path = Path(main_script_path_str)
@@ -168,7 +177,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None:
     }
 
     # Refresh the page configuration
-    _on_pages_changed.send()
+    set_pages(pages)
 
 
 length_captcha = 5

src/common/common.py

@@ -1,17 +1,18 @@
-import json
 import os
-import shutil
 import sys
 import uuid
+import json
 import time
+import psutil
+import shutil
+
+import pandas as pd
+import streamlit as st
+
 from typing import Any
 from pathlib import Path
 from streamlit.components.v1 import html
 
-import streamlit as st
-import pandas as pd
-import psutil
-
 try:
     from tkinter import Tk, filedialog
 
@@ -310,28 +311,25 @@ def render_sidebar(page: str = "") -> None:
         # The main page has workspace switcher
         # Display workspace switcher if workspace is enabled in local mode
         if st.session_state.settings["enable_workspaces"]:
-            with st.expander("🖥️ **Workspaces**"):
-                # Workspaces directory specified in the settings.json
-                if (
-                    st.session_state.settings["workspaces_dir"]
-                    and st.session_state.location == "local"
-                ):
-                    workspaces_dir = Path(
-                        st.session_state.settings["workspaces_dir"],
-                        "workspaces-" + st.session_state.settings["repository-name"],
-                    )
-                else:
-                    workspaces_dir = ".."
-                # Online: show current workspace name in info text and option to change to other existing workspace
-                if st.session_state.location == "local":
+            # Workspaces directory specified in the settings.json
+            if (
+                st.session_state.settings["workspaces_dir"]
+                and st.session_state.location == "local"
+            ):
+                workspaces_dir = Path(
+                    st.session_state.settings["workspaces_dir"],
+                    "workspaces-" + st.session_state.settings["repository-name"],
+                )
+            else:
+                workspaces_dir = ".."
+            # Online: show current workspace name in info text and option to change to other existing workspace
+            if st.session_state.location == "local":
+                with st.expander("🖥️ **Workspaces**"):
                     # Define callback function to change workspace
                     def change_workspace():
                         for key in params.keys():
                             if key in st.session_state.keys():
                                 del st.session_state[key]
-                        st.session_state.workspace = Path(
-                            workspaces_dir, st.session_state["chosen-workspace"]
-                        )
                         st.query_params.workspace = st.session_state["chosen-workspace"]
 
                     # Get all available workspaces as options

src/workflow/CommandExecutor.py

@@ -77,34 +77,85 @@ def run_command(self, command: list[str]) -> None:
         self.logger.log(f"Running command:\n"+' '.join(command)+"\nWaiting for command to finish...", 1)   
         start_time = time.time()
 
-        # Execute the command
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        # Execute the command with real-time output capture
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,  # Line buffered
+            universal_newlines=True
+        )
         child_pid = process.pid
 
         # Record the PID to keep track of running processes associated with this workspace/workflow
         # User can close the Streamlit app and return to a running workflow later
         pid_file_path = self.pid_dir / str(child_pid)
         pid_file_path.touch()
 
-        # Wait for command completion and capture output
-        stdout, stderr = process.communicate()
+        # Real-time output capture
+        self._stream_output(process)
+        
+        # Wait for process completion
+        process.wait()
 
         # Cleanup PID file
         pid_file_path.unlink()
 
         end_time = time.time()
         execution_time = end_time - start_time
-        # Format the logging prefix
+        
+        # Log completion
         self.logger.log(f"Process finished:\n"+' '.join(command)+f"\nTotal time to run command: {execution_time:.2f} seconds", 1)
 
-        # Log stdout if present
-        if stdout:
-            self.logger.log(stdout.decode(), 2)
+        # Check for errors
+        if process.returncode != 0:
+            self.logger.log(f"ERRORS OCCURRED: Process exited with code {process.returncode}", 2)
+
+    def _stream_output(self, process: subprocess.Popen) -> None:
+        """
+        Streams stdout and stderr from a running process in real-time to the logger.
+        This method runs in the workflow process, not the GUI thread, so it's safe to block.
+        
+        Args:
+            process: The subprocess.Popen object to stream from
+        """
+        def read_stdout():
+            """Read stdout in real-time"""
+            try:
+                for line in iter(process.stdout.readline, ''):
+                    if line:
+                        self.logger.log(line.rstrip(), 2)
+                    if process.poll() is not None:
+                        break
+            except Exception as e:
+                self.logger.log(f"Error reading stdout: {e}", 2)
+            finally:
+                process.stdout.close()
+        
+        def read_stderr():
+            """Read stderr in real-time"""
+            try:
+                for line in iter(process.stderr.readline, ''):
+                    if line:
+                        self.logger.log(f"STDERR: {line.rstrip()}", 2)
+                    if process.poll() is not None:
+                        break
+            except Exception as e:
+                self.logger.log(f"Error reading stderr: {e}", 2)
+            finally:
+                process.stderr.close()
+        
+        # Start threads to read stdout and stderr simultaneously
+        stdout_thread = threading.Thread(target=read_stdout, daemon=True)
+        stderr_thread = threading.Thread(target=read_stderr, daemon=True)
+        
+        stdout_thread.start()
+        stderr_thread.start()
 
-        # Log stderr and raise an exception if errors occurred
-        if stderr or process.returncode != 0:
-            error_message = stderr.decode().strip()
-            self.logger.log(f"ERRORS OCCURRED:\n{error_message}", 2)
+        # Wait for both threads to complete
+        stdout_thread.join()
+        stderr_thread.join()
 
     def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> None:
         """

src/workflow/StreamlitUI.py

@@ -995,35 +995,67 @@ def execution_section(self, start_workflow_function) -> None:
         log_level = c1.selectbox(
             "log details", ["minimal", "commands and run times", "all"], key="log_level"
         )
-        if self.executor.pid_dir.exists():
+        
+        # Real-time display options
+        if "log_lines_count" not in st.session_state:
+            st.session_state.log_lines_count = 100
+        
+        log_lines_count = c2.selectbox(
+            "lines to show", [50, 100, 200, 500, "all"],
+            index=1, key="log_lines_select"
+        )
+        if log_lines_count != "all":
+            st.session_state.log_lines_count = log_lines_count
+        
+        pid_exists = self.executor.pid_dir.exists()
+        log_path = Path(self.workflow_dir, "logs", log_level.replace(" ", "-") + ".log")
+        log_exists = log_path.exists()
+
+        if pid_exists:
             if c1.button("Stop Workflow", type="primary", use_container_width=True):
                 self.executor.stop()
                 st.rerun()
         elif c1.button("Start Workflow", type="primary", use_container_width=True):
             start_workflow_function()
-            st.rerun()
-        log_path = Path(self.workflow_dir, "logs", log_level.replace(" ", "-") + ".log")
-        if log_path.exists():
-            if self.executor.pid_dir.exists():
-                with st.spinner("**Workflow running...**"):
-                    with open(log_path, "r", encoding="utf-8") as f:
-                        st.code(
-                            "".join(f.readlines()[-30:]),
-                            language="neon",
-                            line_numbers=False,
-                        )
-                    time.sleep(2)
+            with st.spinner("**Workflow running...**"):
+                time.sleep(1)
                 st.rerun()
-            else:
-                st.markdown(
-                    f"**Workflow log file: {datetime.fromtimestamp(log_path.stat().st_ctime).strftime('%Y-%m-%d %H:%M')} CET**"
-                )
+
+        if log_exists and pid_exists:
+            # Real-time display during execution
+            with st.spinner("**Workflow running...**"):
                 with open(log_path, "r", encoding="utf-8") as f:
-                    content = f.read()
-                    # Check if workflow finished successfully
-                    if not "WORKFLOW FINISHED" in content:
-                        st.error("**Errors occurred, check log file.**")
-                    st.code(content, language="neon", line_numbers=False)
+                    lines = f.readlines()
+                if log_lines_count == "all":
+                    display_lines = lines
+                else:
+                    display_lines = lines[-st.session_state.log_lines_count:]
+                st.code(
+                    "".join(display_lines),
+                    language="neon",
+                    line_numbers=False,
+                )
+                # Faster polling for real-time updates
+                time.sleep(1)
+                st.rerun()
+
+        elif log_exists and not pid_exists:
+            # Static display after completion
+            st.markdown(
+                f"**Workflow log file: {datetime.fromtimestamp(log_path.stat().st_ctime).strftime('%Y-%m-%d %H:%M')} CET**"
+            )
+            with open(log_path, "r", encoding="utf-8") as f:
+                content = f.read()
+            # Check if workflow finished successfully
+            if not "WORKFLOW FINISHED" in content:
+                st.error("**Errors occurred, check log file.**")
+            st.code(content, language="neon", line_numbers=False)
+        elif pid_exists:
+            with st.spinner("**Workflow running...**"):
+                time.sleep(1)
+                st.rerun()
+
+
 
     def results_section(self, custom_results_function) -> None:
         custom_results_function()

src/workflow/WorkflowManager.py

@@ -34,8 +34,6 @@ def start_workflow(self) -> None:
         # Add workflow process id to pid dir
         self.executor.pid_dir.mkdir()
         Path(self.executor.pid_dir, str(workflow_process.pid)).touch()
-        time.sleep(3)
-        st.rerun()
 
     def workflow_process(self) -> None:
         """