adapted icversion

Snakefile

@@ -12,5 +12,5 @@ rule one:
     log:
         "logs/logs-{argcode}"
     shell:
-        "RUN_ONE_INDIR=/tmp/run-one-$RANDOM bash ci-template-cc/make-functions.sh run-one {input} > {log} 2>&1 && touch results/result-{wildcards.argcode}"
+        "RUN_ONE_INDIR=/tmp/run-one-$RANDOM bash ci-template-cc/make-functions.sh run-one {input} > {log} 2>&1; echo $? > results/result-{wildcards.argcode}"
         #"sleep $((RANDOM%300)); RUN_ONE_INDIR=/tmp/run-one-$RANDOM bash ci-template-cc/make-functions.sh run-one {input} > {log} 2>&1 && touch results/result-{wildcards.argcode}"

make-functions.sh

@@ -18,8 +18,8 @@ fi
 
 if [ ${ANNOTATE:-yes}  == "yes" ]; then
 # need to unbuffer this
-    exec 2> >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[31m", $0, "\033[0m"}')
-    exec > >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[0m", $0}')
+exec 2> >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[31m", $0, "\033[0m"; fflush()}')
+exec > >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[0m", $0; fflush()}')
 fi
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -337,21 +337,23 @@ function build-singularity() {
 #    cwltool job.cwl --subcases_pattern="_1" --source_name="Her X-1" --osa_version='OSA11.0' --nscw=10 --ng_sig_limit=2. --systematic_fraction=0.01
 
 function make-args() {
+        icversion=${1:?}
 	for c in $(ls subcases/); do 
 		echo $c
-		python -c 'import yaml; open("args/args-'$c'.yaml", "w").write(yaml.dump({"subcases_pattern": "'$c'", "systematic_fraction": 0.03, "osa_version": "OSA11.0-dev1105", "nscw": 50}))'; 
+		python -c 'import yaml; open("args/args-'$c'.yaml", "w").write(yaml.dump({"subcases_pattern": "'$c'", "systematic_fraction": 0.03, "osa_version": "OSA11.0-'${icversion}'", "nscw": 50}))'; 
 	done
 }
 
 function submit-all-scw() {
     icversion=${1:?}   
+    nscwmax=${2:-5000}
 
     oda-node version -v || {
         echo -e "\033[31mfailed to connect to ODAHUB\033[0m"
         exit 1
     }
 
-    for scw in $(cat */subcases/*/*scw*txt subcases/*/*scw*txt | shuf |  awk '/^[0-9]{12}$/' ) ; do 
+    for scw in $(cat */subcases/*/*scw*txt subcases/*/*scw*txt | shuf |  awk '/^[0-9]{12}$/' | head -n$nscwmax) ; do 
         echo $scw
         
         oda-node ask \
@@ -360,7 +362,7 @@ function submit-all-scw() {
                 -m git://findic/staging-1-3-icversion \
                 -m git://ddosa11/staging-1-3 \
                 -m git://useresponse/staging-1-3-osa11 \
-                -a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-3}')' \
+                -a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
                 -a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
         oda-node ask \
             ii_spectra_extract \
@@ -368,7 +370,37 @@ function submit-all-scw() {
                 -m git://findic/staging-1-3-icversion \
                 -m git://ddosa11/staging-1-3 \
                 -m git://useresponse/staging-1-3-osa11 \
-                -a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-3}')' \
+                -a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
+                -a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
+                #-a 'useresponse.CompressEBins(use_factor=4)' \
+    done
+}
+
+function submit-all-cases() {
+    icversion=${1:?}   
+    nmax=${2:?}
+
+    oda-node version -v || {
+        echo -e "\033[31mfailed to connect to ODAHUB\033[0m"
+        exit 1
+    }
+
+    for subcase in $(ls subcases/ | shuf -n $nmax) ; do 
+        echo $subcase
+
+        scws=$(cat subcases/$subcase/*scw*txt | awk '{printf $1".001,"}')
+        
+        oda-node ask \
+            ISGRISpectraSum \
+                -m git://ddosa/staging-1-3 \
+                -m git://findic/staging-1-3-icversion \
+                -m git://ddosa11/staging-1-3 \
+                -m git://useresponse/staging-1-3-osa11 \
+                -m git://process_isgri_spectra/staging-1-3-osa11 \
+                -a 'process_isgri_spectra.ISGRISpectraSum(use_extract_all=True)' \
+                -a 'ddosa.IDScWList(use_scwid_list="'$scws'".strip(",").split(","))' \
+                -a 'process_isgri_spectra.ScWSpectraList(input_scwlist=ddosa.IDScWList)' \
+                -a 'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
                 -a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
                 #-a 'useresponse.CompressEBins(use_factor=4)' \
     done

odacc/cli.py

@@ -46,11 +46,14 @@ def cli(ctx, debug, change_dir=None):
     if change_dir is None:
         change_dir = os.getcwd()
 
-    obj["change_dir"] = change_dir
+    try:
+        obj["change_dir"] = change_dir
 
-    obj["oda_yaml"] = yaml.load(open(os.path.join(change_dir, "oda.yaml")), Loader=yaml.SafeLoader)
+        obj["oda_yaml"] = yaml.load(open(os.path.join(change_dir, "oda.yaml")), Loader=yaml.SafeLoader)
 
-    logging.debug("loaded: %s", obj['oda_yaml'])
+        logging.debug("loaded: %s", obj['oda_yaml'])
+    except Exception as e:
+        logging.warning("can not load oda.yaml: not test case based call")
 
 
 @cli.command()
@@ -109,5 +112,30 @@ def version(obj, check):
             print(f"version matches in {check}")
 
 
+@cli.command()
+@click.argument("bucket")
+def annotate_explain_bucket(bucket):
+    import odakb.datalake
+
+    m, d = odakb.datalake.restore(bucket, return_metadata=True)
+
+    print(m)
+
+    odakb.sparql.insert(f'oda:{bucket} a oda:ccbucket; oda:bucket "{bucket}"')
+
+    funcbase = m["query"].replace("http://", "").replace(".io/", "").replace("/", "-") 
+    func = funcbase + m['version']
+
+    x  = f'oda:{bucket} oda:curryingOf oda:{func} .'
+    x += f'oda:{func} oda:curryingOf oda:{funcbase} .'
+    print(x)
+    odakb.sparql.insert(x)
+
+    for k, v in m['kwargs'].items():
+        x = f'oda:{bucket} oda:curryied_input_{k} "{v}"'
+        print(">>", x)
+        odakb.sparql.insert(x)
+
+
 if __name__ == "__main__":
     cli()