Feature/scripts

.github/workflows/python-package-conda.yml

@@ -43,7 +43,7 @@ jobs:
           cd $GITHUB_WORKSPACE
           flake8 --exclude src/hydrodiy/data/tests src/hydrodiy/data
           flake8 --exclude src/hydrodiy/gis/tests,src/hydrodiy/gis/data src/hydrodiy/gis
-          flake8 --exclude src/hydrodiy/io/tests,src/hydrodiy/io/script_template_plot.py,src/hydrodiy/io/script_template_full.py,src/hydrodiy/io/script_template_default.py,src/hydrodiy/io/script_template_cmd.py src/hydrodiy/io
+          flake8 --exclude src/hydrodiy/io/tests,src/hydrodiy/io/script_template_plot.py,src/hydrodiy/io/script_template_minimal.py,src/hydrodiy/io/script_template_cmd.py src/hydrodiy/io
           flake8 --exclude src/hydrodiy/plot/tests src/hydrodiy/plot
           flake8 --exclude src/hydrodiy/stat/tests,src/hydrodiy/stat/data src/hydrodiy/stat
 

.gitignore

@@ -31,7 +31,7 @@ src/hydrodiy/gis/tests/*.shp
 src/hydrodiy/gis/tests/*.shx
 src/hydrodiy/plot/tests/images/
 src/hydrodiy/stat/tests/__pycache__/
-src/hydrodiy/io/tests/script_test*
+src/hydrodiy/io/tests/run_scripts/*
 src/hydrodiy/gis/tests/grid_test.bil
 src/hydrodiy/data/c_hydrodiy_data.c
 src/hydrodiy/stat/c_hydrodiy_stat.c

src/hydrodiy/io/iutils.py

@@ -9,11 +9,11 @@
 import logging
 import stat
 
-SCRIPT_TYPES = ["default", "full", "cmd", "plot"]
+SCRIPT_TYPES = ["minimal", "cmd", "plot"]
 
 
 def script_template(filename, comment,
-                    stype="default",
+                    stype="cmd",
                     author=None,
                     fout=None, fdata=None, fimg=None):
     """ Write a script template
@@ -85,21 +85,15 @@ def script_template(filename, comment,
 
     if fout is None:
         fout = "froot / \"outputs\""
-        if stype != "cmd":
-            fout += "\nfout.mkdir(exist_ok=True)\n"
     txt = re.sub("\\[FOUT\\]", fout, txt)
 
     if fdata is None:
         fdata = "froot / \"data\""
-        if stype != "cmd":
-            fdata += "\nfdata.mkdir(exist_ok=True)\n"
     txt = re.sub("\\[FDATA\\]", fdata, txt)
 
     if stype == "plot":
         if fimg is None:
             fimg = "froot / \"images\""
-            if stype != "cmd":
-                fimg += "\nfimg.mkdir(exist_ok=True)\n"
         txt = re.sub("\\[FIMG\\]", fimg, txt)
     else:
         txt = re.sub("fimg = \\[FIMG\\]", "", txt)

src/hydrodiy/io/script_template_cmd.py

@@ -31,29 +31,37 @@ def get_script_paths(config):
     froot = [FROOT]
     fdata = [FDATA]
     fout = [FOUT]
-    flogs = froot / "logs"
+    flogs = froot / "logs" / source_file.stem
 
-    if debug:
+    if config.debug:
         fout = flogs / fout.stem
 
-    SP = namedtuple("ScriptPaths",
-                    ["source_file", "froot", "fdata", "fout", "flogs"])
-    script_paths = SP(source_file, froot, fdata, fout, flogs)
+    ScriptPaths = namedtuple("ScriptPaths",
+                             ["source_file", "basename",
+                              "froot", "fdata", "fout", "flogs"])
+    script_paths = ScriptPaths(source_file, source_file.stem,
+                               froot, fdata, fout, flogs)
+    if config.create_folders:
+        flogs.mkdir(exist_ok=True, parents=True)
 
-    for pa in script_paths:
-        if pa.is_file():
-            continue
-        pa.mkdir(exist_ok=True)
+        fout.mkdir(exist_ok=True, parents=True)
+
+        # Clean output folder if needed
+        cext = config.clean_folders_extension
+        if cext != "":
+            for f in fout.glob("*." + cext):
+                f.unlink()
 
     return script_paths
 
 
 def get_logger(config, script_paths):
-    basename = script_paths.source_file.stem
+    basename = script_paths.basename
     fl = script_paths.flogs / f"{basename}.log"
     logger = iutils.get_logger(basename, flog=fl, console=True)
-    logger.log_dict(vars(args), "Command line arguments")
-    logger.started()
+    logger.log_dict(config._asdict(), "Configuration")
+    logger.info("", nret=1)
+
     return logger
 
 
@@ -74,7 +82,7 @@ def get_data(config, script_paths, logger):
         stations = allstations.iloc[idx, :]
 
     nstations = len(stations)
-    logger.info(f"Dealing with {nstations} stations.")
+    logger.info(f"Found {nstations} stations.")
 
     data = namedtuple("Data", ["stations"])(stations)
 
@@ -83,7 +91,7 @@ def get_data(config, script_paths, logger):
 
 def process(config, script_paths, logger, data):
     nstations = len(data.stations)
-    logger.info(f"Start processing - Debug={config.debug}", nret=1)
+    logger.info(f"Start processing", nret=1)
 
     for isite, (stationid, sinfo) in enumerate(data.stations.iterrows()):
         ctxt = f"{stationid} ({isite+1}/{nstations})"
@@ -113,22 +121,26 @@ def process(config, script_paths, logger, data):
     overwrite = args.overwrite
     debug = args.debug
     sitepattern = args.sitepattern
-
-    clear = False
-    nbatch = 1
-    ibatch = 0
-
-    CF = namedtuple("Config",
-                    ["version", "taskid",
-                     "overwrite", "debug",
-                     "sitepattern", "clear",
-                     "nbatch", "ibatch"])
-    config = CF(version, taskid, overwrite, debug,
-                sitepattern, clear, nbatch, ibatch)
-
-    # Get data
+    nbatch = 4
+    ibatch = -1 if debug else taskid
+    create_folders = True
+    clean_folders_extension = ""
+
+    Config = namedtuple("Config",
+                        ["version", "taskid", "overwrite",
+                         "debug", "create_folders",
+                         "clean_folders_extension",
+                         "sitepattern", "nbatch", "ibatch"])
+    config = Config(version, taskid, overwrite,
+                    debug, create_folders,
+                    clean_folders_extension,
+                    sitepattern, nbatch, ibatch)
+
+    # Baseline
     script_paths = get_script_paths(config)
     logger = get_logger(config, script_paths)
+
+    # Data
     data = get_data(config, script_paths, logger)
 
     # Process