Unverified Commit 02a777d0 authored by Volodymyr SAVCHENKO's avatar Volodymyr SAVCHENKO
Browse files

adapted icversion

parent 5029625a
......@@ -12,5 +12,5 @@ rule one:
log:
"logs/logs-{argcode}"
shell:
"RUN_ONE_INDIR=/tmp/run-one-$RANDOM bash ci-template-cc/make-functions.sh run-one {input} > {log} 2>&1 && touch results/result-{wildcards.argcode}"
"RUN_ONE_INDIR=/tmp/run-one-$RANDOM bash ci-template-cc/make-functions.sh run-one {input} > {log} 2>&1; echo $? > results/result-{wildcards.argcode}"
#"sleep $((RANDOM%300)); RUN_ONE_INDIR=/tmp/run-one-$RANDOM bash ci-template-cc/make-functions.sh run-one {input} > {log} 2>&1 && touch results/result-{wildcards.argcode}"
......@@ -18,8 +18,8 @@ fi
if [ ${ANNOTATE:-yes} == "yes" ]; then
# need to unbuffer this
exec 2> >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[31m", $0, "\033[0m"}')
exec > >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[0m", $0}')
exec 2> >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[31m", $0, "\033[0m"; fflush()}')
exec > >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[0m", $0; fflush()}')
fi
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
......@@ -337,21 +337,23 @@ function build-singularity() {
# cwltool job.cwl --subcases_pattern="_1" --source_name="Her X-1" --osa_version='OSA11.0' --nscw=10 --ng_sig_limit=2. --systematic_fraction=0.01
function make-args() {
icversion=${1:?}
for c in $(ls subcases/); do
echo $c
python -c 'import yaml; open("args/args-'$c'.yaml", "w").write(yaml.dump({"subcases_pattern": "'$c'", "systematic_fraction": 0.03, "osa_version": "OSA11.0-dev1105", "nscw": 50}))';
python -c 'import yaml; open("args/args-'$c'.yaml", "w").write(yaml.dump({"subcases_pattern": "'$c'", "systematic_fraction": 0.03, "osa_version": "OSA11.0-'${icversion}'", "nscw": 50}))';
done
}
function submit-all-scw() {
icversion=${1:?}
nscwmax=${2:-5000}
oda-node version -v || {
echo -e "\033[31mfailed to connect to ODAHUB\033[0m"
exit 1
}
for scw in $(cat */subcases/*/*scw*txt subcases/*/*scw*txt | shuf | awk '/^[0-9]{12}$/' ) ; do
for scw in $(cat */subcases/*/*scw*txt subcases/*/*scw*txt | shuf | awk '/^[0-9]{12}$/' | head -n$nscwmax) ; do
echo $scw
oda-node ask \
......@@ -360,7 +362,7 @@ function submit-all-scw() {
-m git://findic/staging-1-3-icversion \
-m git://ddosa11/staging-1-3 \
-m git://useresponse/staging-1-3-osa11 \
-a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-3}')' \
-a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
-a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
oda-node ask \
ii_spectra_extract \
......@@ -368,7 +370,37 @@ function submit-all-scw() {
-m git://findic/staging-1-3-icversion \
-m git://ddosa11/staging-1-3 \
-m git://useresponse/staging-1-3-osa11 \
-a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-3}')' \
-a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
-a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
#-a 'useresponse.CompressEBins(use_factor=4)' \
done
}
function submit-all-cases() {
icversion=${1:?}
nmax=${2:?}
oda-node version -v || {
echo -e "\033[31mfailed to connect to ODAHUB\033[0m"
exit 1
}
for subcase in $(ls subcases/ | shuf -n $nmax) ; do
echo $subcase
scws=$(cat subcases/$subcase/*scw*txt | awk '{printf $1".001,"}')
oda-node ask \
ISGRISpectraSum \
-m git://ddosa/staging-1-3 \
-m git://findic/staging-1-3-icversion \
-m git://ddosa11/staging-1-3 \
-m git://useresponse/staging-1-3-osa11 \
-m git://process_isgri_spectra/staging-1-3-osa11 \
-a 'process_isgri_spectra.ISGRISpectraSum(use_extract_all=True)' \
-a 'ddosa.IDScWList(use_scwid_list="'$scws'".strip(",").split(","))' \
-a 'process_isgri_spectra.ScWSpectraList(input_scwlist=ddosa.IDScWList)' \
-a 'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
-a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
#-a 'useresponse.CompressEBins(use_factor=4)' \
done
......
......@@ -46,11 +46,14 @@ def cli(ctx, debug, change_dir=None):
if change_dir is None:
change_dir = os.getcwd()
obj["change_dir"] = change_dir
try:
obj["change_dir"] = change_dir
obj["oda_yaml"] = yaml.load(open(os.path.join(change_dir, "oda.yaml")), Loader=yaml.SafeLoader)
obj["oda_yaml"] = yaml.load(open(os.path.join(change_dir, "oda.yaml")), Loader=yaml.SafeLoader)
logging.debug("loaded: %s", obj['oda_yaml'])
logging.debug("loaded: %s", obj['oda_yaml'])
except Exception as e:
logging.warning("can not load oda.yaml: not test case based call")
@cli.command()
......@@ -109,5 +112,30 @@ def version(obj, check):
print(f"version matches in {check}")
@cli.command()
@click.argument("bucket")
def annotate_explain_bucket(bucket):
import odakb.datalake
m, d = odakb.datalake.restore(bucket, return_metadata=True)
print(m)
odakb.sparql.insert(f'oda:{bucket} a oda:ccbucket; oda:bucket "{bucket}"')
funcbase = m["query"].replace("http://", "").replace(".io/", "").replace("/", "-")
func = funcbase + m['version']
x = f'oda:{bucket} oda:curryingOf oda:{func} .'
x += f'oda:{func} oda:curryingOf oda:{funcbase} .'
print(x)
odakb.sparql.insert(x)
for k, v in m['kwargs'].items():
x = f'oda:{bucket} oda:curryied_input_{k} "{v}"'
print(">>", x)
odakb.sparql.insert(x)
if __name__ == "__main__":
cli()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment