update log filehandler

evalscope/api/benchmark/adapters/default_data_adapter.py

@@ -768,3 +768,6 @@ def generate_report(self, scores: Dict[str, List[AggScore]], model_name: str, ou
     def finalize(self, *args, **kwargs):
         # Finalize the evaluation process
         self.sandbox_finalize(*args, **kwargs)
+        # Release dataset memory after evaluation to avoid accumulation across benchmarks
+        self.test_dataset = None
+        self.fewshot_dataset = None

evalscope/benchmarks/ifeval/instructions_util.py

@@ -19,6 +19,8 @@
 import random
 import re
 
+from evalscope.utils.resource_utils import check_nltk_data
+
 RANK = os.environ.get('LOCAL_RANK', '0')
 
 WORD_LIST = [
@@ -1654,7 +1656,11 @@ def count_words(text):
 
 @functools.lru_cache(maxsize=None)
 def _get_sentence_tokenizer():
-    return nltk.data.load('nltk:tokenizers/punkt/english.pickle')
+    check_nltk_data('punkt_tab')
+    try:
+        return nltk.data.load('tokenizers/punkt_tab/english.pickle')
+    except LookupError:
+        return nltk.data.load('nltk:tokenizers/punkt/english.pickle')
 
 
 def count_sentences(text):

evalscope/run.py

@@ -159,9 +159,12 @@ def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
             unit='benchmark',
             logger=logger,
         ) as pbar:
-            for evaluator in pbar:
+            for i, evaluator in enumerate(pbar):
                 res_dict = evaluator.eval()
                 eval_results[evaluator.benchmark_name] = res_dict
+                # Release evaluator immediately after eval to avoid
+                # accumulating all benchmark objects in memory simultaneously
+                evaluators[i] = None
 
     # Make overall report
     try:

evalscope/utils/logger.py

@@ -20,6 +20,9 @@
 plain_simple_formatter = logging.Formatter(simple_format, datefmt=data_format)
 
 DEFAULT_LEVEL = logging.DEBUG if os.getenv('EVALSCOPE_LOG_LEVEL', 'INFO') == 'DEBUG' else logging.INFO
+# Use ReopenFileHandler on OSS/FUSE mounts so each record is visible immediately;
+# fall back to the standard FileHandler on local filesystems.
+_USE_OSS = os.getenv('USE_OSS', '0') == '1'
 
 logging.basicConfig(format=simple_format, level=logging.INFO, force=True)
 
@@ -48,6 +51,44 @@ def warning_once(self, msg, *args, **kwargs):
     self.warning(msg)
 
 
+class ReopenFileHandler(logging.FileHandler):
+    """FileHandler that closes the file after every emit.
+
+    On OSS/FUSE-mounted filesystems the FUSE driver only uploads data when
+    the file descriptor is closed.  By reopening on each record this handler
+    ensures every log line is visible on OSS in near real-time.
+
+    Thread safety: emit() is invoked inside Handler.handle() which already
+    holds self.lock, so no extra locking is needed here.
+    """
+
+    def __init__(self, filename: str, mode: str = 'a', encoding: str = 'utf-8'):
+        # delay=True: skip opening the file in __init__; we open it ourselves in emit()
+        super().__init__(filename, mode=mode, encoding=encoding, delay=True)
+        self._first_write = True
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Open → format+write → flush → close for every log record."""
+        # After the first write switch to append so we never truncate
+        if not self._first_write:
+            self.mode = 'a'
+        self.stream = self._open()
+        try:
+            logging.StreamHandler.emit(self, record)
+        finally:
+            if self.stream is not None:
+                try:
+                    self.stream.flush()
+                    self.stream.close()
+                finally:
+                    self.stream = None
+            self._first_write = False
+
+
+# Module-level handler class: resolved once at import time from the USE_OSS env var.
+FILE_HANDLER_CLS = ReopenFileHandler if _USE_OSS else logging.FileHandler
+
+
 def get_logger(
     log_file: Optional[str] = None,
     name: Optional[str] = None,
@@ -108,7 +149,7 @@ def get_logger(
     handlers = [stream_handler]
 
     if is_worker0 and log_file is not None:
-        file_handler = logging.FileHandler(log_file, file_mode, encoding='utf-8')
+        file_handler = FILE_HANDLER_CLS(log_file, mode=file_mode, encoding='utf-8')
         handlers.append(file_handler)
 
     for handler in handlers:
@@ -182,7 +223,7 @@ def add_file_handler_if_needed(
         except Exception:
             pass
 
-    file_handler = logging.FileHandler(target_path, file_mode, encoding='utf-8')
+    file_handler = FILE_HANDLER_CLS(target_path, mode=file_mode, encoding='utf-8')
     file_handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
     file_handler.setLevel(log_level)
     logger.addHandler(file_handler)

tests/benchmark/test_eval.py

@@ -54,7 +54,7 @@ def test_gsm8k(self):
         dataset_args = {
             'few_shot_num': 0,
         }
-        self._run_dataset_test('gsm8k', dataset_args=dataset_args, limit=5, debug=True, eval_batch_size=1)
+        self._run_dataset_test('gsm8k', dataset_args=dataset_args, limit=5, debug=False, eval_batch_size=1, use_cache='outputs/20260414_113853')
 
     def test_gsm8k_pass_at_k(self):
         """Test GSM8K math reasoning dataset with Pass@k metric."""