make-functions.sh 18.1 KB
Newer Older
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
1
2
#!/bin/bash

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
3
# useful defaults
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
4
5
export ODA_SPARQL_ROOT=${ODA_SPARQL_ROOT:-https://www.astro.unige.ch/cdci/astrooda/dispatch-data/gw/odakb}
#export ODA_SPARQL_ROOT=${ODA_SPARQL_ROOT:-http://fuseki.internal.odahub.io/dataanalysis}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
6

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
7
8
export PATH=$HOME/.local/bin:$PATH

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
9
10
11
12
13
14
15
16
17
18
19
20
function get_pypi_version() {
    pkg_name=${1:?}

    curl  https://pypi.org/pypi/${pkg_name}/json | python -c 'import sys, json; print(json.load(sys.stdin)["info"]["version"])'
}

function ensure_latest_pypi() {
    pkg_name=${1:?}

    [ "$(get_pypi_version $pkg_name)" == $(python -c 'import pkg_resources; print(pkg_resources.require("nb2workflow")[0].version)' ) ] || pip install $pkg_name
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
21
22
23
24
25
26
27
28
if [ -s oda.yaml ]; then
    export IN_TEST_CASE=yes
    echo -e "\033[32mfound test case oda.yaml\033[0m"
else
    export IN_TEST_CASE=no
    echo -e "\033[34mNOT found test case oda.yaml, no-test-case mode\033[0m"
fi

29
30
31
32
if [ ${DEBUG:-no} == "yes" ]; then
    set -x
fi

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
33
34
35

if [ ${ANNOTATE:-yes}  == "yes" ]; then
# need to unbuffer this
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
36
37
exec 2> >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[31m", $0, "\033[0m"; fflush()}')
exec > >(awk '{print "\033[90m", strftime("%Y-%m-%dT%H:%M:%S"), "\033[0m", $0; fflush()}')
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
38
fi
39

ODA CI Bot's avatar
ODA CI Bot committed
40
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
41

ODA CI Bot's avatar
ODA CI Bot committed
42
43
echo -e "\033[32mfound ci-template-cc location in $DIR\033[0m"

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
44
function upgrade-oda-python() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
45
46
    t0=$(date +%s)

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
47
    echo "will update osa python"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
48
49
    PIP="python -m pip"
    echo "selected pip: $PIP"
50

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
51
52
53
54
55
56
57
    # because pip is just too slow
    if python -c '
import sys
import importlib as imp
if imp.util.find_spec("yaml") is None:
    sys.exit(1)
'; then
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
58
        echo -e '\033[32m yaml found\033[0m'
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
59
    else
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
60
        echo -e '\033[31m no yaml, will install\033[0m'
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
61
62
63
        $PIP install --upgrade pyyaml --user || pip install --upgrade pyyaml \
            || { echo -e "\033[31missue updating python module of pyyaml from $DIR!\033[0m"; exit 1; }
    fi
ODA CI Bot's avatar
ODA CI Bot committed
64

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
65
    if [ -s $DIR/setup.py ]; then
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
66
        echo -e "\033[32mfound $DIR/setup.py\033[0m"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
67
68
69
70
        oda-cc version -c $DIR || {
            $PIP install --upgrade ${DIR} --user || pip install --upgrade ${DIR} \
                || { echo -e "\033[31missue updating python module for ci-tempalte-cc from $DIR!\033[0m"; exit 1; }
        }
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
71

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
72
        ensure_latest_pypi nb2workflow
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
73
74
75
    else
        echo -e "\033[34mNOTE: no python module in script directory - probably system-wide install, not verifying"
    fi
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
76

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
77
    NB2WORKFLOW_VERSION=$(nb2workflow-version)
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
78
79
80

    tdone=$(date +%s)
    echo "\033[31mspent $((tdone - t0)) seconds in upgrade-oda-python\033[0m"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
81
}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
82

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
83
84
if [ "$IN_TEST_CASE" == "yes" ]; then
    upgrade-oda-python
85

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
86
87
88
89
90
91
    if oda-cc get uri_base unset; then
        echo "found operable oda-cc from ci-template-cc"
    else
        echo -e "\033[31mNOT found operable oda-cc from ci-template-cc\033[0m"
        exit 1
    fi
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
92

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
93
    SOURCE_NAME=$( oda-cc get metadata.source_short_name unnamed )
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
94
    TEST_CASE_NAME=$( oda-cc get metadata.test_case_short_name unnamed )
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
95
96
97
    CC_BASE_IMAGE=$( oda-cc get cc_base_image integralsw/osa-python:11.1-3-g87cee807-20200410-144247-refcat-43.0-heasoft-6.27.2-python-3.8.2 )
    ROOT_NB=$( oda-cc get root_notebook verify.ipynb )
fi
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
98

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
99
PORT=8998
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
100

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
101
DOCKER_REGISTRY=faircrosscalibration
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
102
#DOCKER_REGISTRY=admin.reproducible.online
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
103

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
104

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
105
106
107
#mkfile_path = $(abspath $(lastword $(MAKEFILE_LIST)))
#current_dir = $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))
#current_version := $(shell cd $(current_dir); git describe --tags)
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
108

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
109
    
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
110
IMAGE="$DOCKER_REGISTRY/cc-$TEST_CASE_NAME:$( git describe --always --tags)-$NB2WORKFLOW_VERSION"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
111
IMAGE=$(echo $IMAGE | tr '[:upper:]' '[:lower:]')
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
112

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
113
114
SINGULARITY_IMAGE=${IMAGE//\//}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
115
116
117
118
echo "req_nb2w: $REQ_NB2W"
echo "image: $IMAGE"

function pull() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
119
120
121
    echo $NB2WORKFLOW_VERSION
    docker pull $IMAGE || true
    touch job.cwl
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
122
123
124
}

function build() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
125
    [ -s pip.conf ] ||  touch pip.conf
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
126
    export DOCKER_BUILDKIT=1
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
127
128
129
130
131
132
133
134
    set -x
    #docker pull $IMAGE && 
    ( 
        nb2worker ./ --build \
            --tag-image $IMAGE \
            --job \
            --from ${CC_BASE_IMAGE:?} \
            --store-dockerfile /tmp/Dockerfile-auto \
135
136
	    --extra-docker-commands='ARG ASTRO_GITLAB_PIP_TOKEN;RUN echo ${ASTRO_GITLAB_PIP_TOKEN}' \
	    --extra-build-arguments="{\"ASTRO_GITLAB_PIP_TOKEN\": \"${ASTRO_GITLAB_PIP_TOKEN}\"}" \
137
            --docker-run-prefix='mkdir -pv /home/oda; export HOME_OVERRRIDE=/home/oda; source /init.sh; ' \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
138
139
140
141
            --docker-command='id; export HOME_OVERRRIDE=/tmp; mkdir -pv $HOME_OVERRRIDE; source /init.sh; source /etc/bashrc; oda-kb-eval /repo/'$ROOT_NB' $@' 
        ls -ltr *cwl
    )
    set +x
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
142
    touch job.cwl
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
143
144
145
146
}


function push() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
147
    docker push $IMAGE
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
148
149
150
}

function clean() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
151
    for nb in *ipynb; do 
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
152
153
154
155
156
            jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace $nb; 
        done
}

function rm() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
157
    docker rm -f $TEST_CASE_NAME || true
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
158
159
}

160

ODA CI Bot's avatar
ODA CI Bot committed
161
function ensure_minio() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
162
163
    if [ -z $MINIO_KEY ]; then
        echo -e "\033[31mWARNING, MINIO_KEY not set\033[0m"
ODA CI Bot's avatar
ODA CI Bot committed
164
        if [ ${LOCAL_ONLY:-no} == "yes" ]; then
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
165
            echo -e "\033[33mWARNING, LOCAL_ONLY=\"${LOCAL_ONLY}\", so MINIO_KEY not set is allowed\033[0m"
ODA CI Bot's avatar
ODA CI Bot committed
166
        else
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
167
            echo -e "\033[31mERROR, LOCAL_ONLY=\"${LOCAL_ONLY}\" not \"yes\", but MINIO_KEY not set - exiting\033[0m"
ODA CI Bot's avatar
ODA CI Bot committed
168
169
170
171
172
            exit 1
        fi
    fi
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
173
function run() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
174
175
  # this will listen with a notebook
  
176
177
    echo -e "\e[34mMounting\e[0m $PWD will be mounted in to use the code, and jupyter workdir "

ODA CI Bot's avatar
ODA CI Bot committed
178
179
180
181
182
183
    if [ ${MOUNT_SSH_FLAG:=no} == "yes" ]; then
        extra="-v /${HOME:?HOME variable is not set?}/.ssh:/home/oda/.ssh:ro"
        echo -e "\033[32mYES mounting ssh keys\033[0m: will rely on ssh when possible"
    else
        echo -e "\033[31mNOT mounting ssh keys\033[0m: will rely on https, please be sure sources are synchronous"
    fi
184

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
185

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
186
    docker rm -f $TEST_CASE_NAME || true
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
187
188
189
190
191
    docker run --entrypoint cat $IMAGE  /etc/passwd > passwd
    < passwd sed 's/1000/'$(id -u)'/' > passwd.new

    [ -s passwd.new ] || exit 1

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
192
193
    echo -e "\033[31mODA_SPARQL_ROOT\033[0m=\033[32m${ODA_SPARQL_ROOT}\033[0m"

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
194
    mkdir -pv /tmp/tmpcode-$TEST_CASE_NAME; \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
195
        docker run --rm -it \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
196
                --user $(id -u) \
197
		-e ODA_SPARQL_ROOT \
198
                -p $PORT:$PORT \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
199
200
201
202
                -v $PWD/passwd.new:/etc/passwd \
                -v $PWD:/repo \
                -v $PWD:/home/jovyan \
                -v $PWD:/home/integral \
ODA CI Bot's avatar
ODA CI Bot committed
203
                -e JENA_PASSWORD \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
204
                -e MINIO_KEY \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
205
                --name $TEST_CASE_NAME \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
206
                --entrypoint  bash $IMAGE \
ODA CI Bot's avatar
ODA CI Bot committed
207
                $extra \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
208
                -c "
209
                    export HOME_OVERRRIDE=/home/jovyan
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
210
211
                    export HOME=/home/jovyan
                    source /init.sh
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
212
213
                    echo $MINIO_KEY > /tmp/home-run/.minio-key
                    chmod 400 /tmp/home-run/.minio-key
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
214
                    cd /repo
215
                    jupyter notebook --ip=0.0.0.0 --no-browser --port=$PORT"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
216
}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
217
218
219
220
221
222
        
function copy-user-id() {
	# in case you do not know, there are no passwords in /etc/passwd
	docker run --entrypoint cat $IMAGE  /etc/passwd > passwd
		< passwd sed 's/1000/'$(id -u)'/' > passwd.new
}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
223

224
# TODO: this logic should be within oda eval 
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
225
function run-one() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
226
227
	set -e
	NBARGS_YAML=${1:-}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
228

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
229
230
231
232
233
234
235
	if [ "$NBARGS_YAML" == "" ]; then
	    echo "NBARGS_YAML empty: will use variable"
	else
	    echo "NBARGS_YAML NOT empty"
	    export NBARGS_PYDICT="$(python -c 'import yaml; print(yaml.load(open("'$NBARGS_YAML'"), Loader=yaml.SafeLoader))')"
	fi
	set +e
ODA CI Bot's avatar
ODA CI Bot committed
236

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
237
238
239
240
241
242
243
        provided_pars_suffix=$(oda-cc format -e NBARGS_PYDICT AUTO)

	echo -e "\033[32m will run-one in ${RUN_ONE_INDIR:=$PWD}\033[0m"
	mkdir -pv $RUN_ONE_INDIR
	cp -fv oda.yaml *.ipynb $RUN_ONE_INDIR
	cd $RUN_ONE_INDIR

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
244
        cname=cc-ci-run-$TEST_CASE_NAME-$provided_pars_suffix
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
245
        ensure_minio
246

ODA CI Bot's avatar
ODA CI Bot committed
247
        if [ ${MOUNT_SSH_FLAG:=no} == "yes" ]; then
248
            extra="-v /${HOME:?HOME variable is not set?}/.ssh:/home/oda/.ssh:ro"
ODA CI Bot's avatar
ODA CI Bot committed
249
250
251
252
            echo -e "\033[32mYES mounting ssh keys\033[0m: will rely on ssh when possible"
        else
            echo -e "\033[31mNOT mounting ssh keys\033[0m: will rely on https, please be sure sources are synchronous"
        fi
ODA CI Bot's avatar
nb2run    
ODA CI Bot committed
253

ODA CI Bot's avatar
ODA CI Bot committed
254
        echo -e "\033[33mnotebooks to run set by NB2RUN (defaults to root notebook $ROOT_NB) ${NB2RUN:=$ROOT_NB}\033[0m"
ODA CI Bot's avatar
nb2run    
ODA CI Bot committed
255
256
        export NB2RUN

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
257
258
	docker rm -f $cname || echo "can not remove $cname - this is ok"

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
259
        docker run \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
260
            --name $cname \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
261
            -e NBARGS_PYDICT \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
262
            -e CI_JOB_TOKEN \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
263
            -e MINIO_KEY \
ODA CI Bot's avatar
ODA CI Bot committed
264
            -e JENA_PASSWORD \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
265
            -e MINIO_USER=${MINIO_USER:-$USER} \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
266
	    -e ODA_SPARQL_ROOT \
ODA CI Bot's avatar
ODA CI Bot committed
267
            $extra \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
268
                --entrypoint  bash $IMAGE \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
269
                -c '''
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
270
                      export PYTHONUNBUFFERED=1
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
271

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
272
                      mkdir -pv /tmp/output
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
273
274

                      mkdir -pv /tmp/home-run
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
275
                      chmod 700 /tmp/home-run
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
276
                      export HOME_OVERRRIDE=/tmp/home-run
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
277
                      
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
278
279
                      source /init.sh

ODA CI Bot's avatar
ODA CI Bot committed
280
281
282
283
284
285
286
287
                      if [ "'$MOUNT_SSH_FLAG'" == "yes" ]; then
                          if git clone git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-nustar-reference.git /tmp/test-clone; then
                              echo -e "\033[32mSUCCESS\033[0mfully cloned private repo from gitlab";
                          else
                              echo -e "\033[31mFAILED033[0m to cloned private repo from gitlab";
                              echo "make sure that your home contains ssh keys"
                              exit 1
                          fi
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
288
                      else
ODA CI Bot's avatar
ODA CI Bot committed
289
                          echo "skipping ssh check: ssh will not be used"
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
290
291
                      fi

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
292
293
                      git clone /repo /tmp/home-run/repo
                      cd /tmp/home-run/repo
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
294
		      git remote set-url origin $(cd /repo; git remote -v | awk "{print \$2}" | head -1)
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
295

296
                      echo "${NBARGS_PYDICT:-{\}}" > nbargs.py
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
297
                      cat nbargs.py
298

299

300
301
302
303
304
305
306
307
308
309
310
311
312
                      nb="'$NB2RUN'"
                      set -ex
                      python -c "import oda, yaml, ast, json;\
                                 md, d = oda.evaluate(
                                         \"kb\", 
                                         yaml.safe_load(open(\"oda.yaml\"))[\"uri_base\"], 
                                         nbname=\""${nb//.ipynb/}"\", 
                                         **ast.literal_eval(open(\"nbargs.py\").read()) or {}, 
                                         _write_files=True,
                                         _return_metadata=True,
                                     );\
                                 json.dump(d, open(\"cwl.output.json\", \"w\"));\
                                 json.dump(md, open(\"metadata.json\", \"w\"))
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
313
                                 " 2>&1  #| cut -c1-400 | tee ${nb//.ipynb/_output.log}  
314
315
316
317
318
319
320
321
                      ls -l cwl.output.json
                      cp -fv cwl.output.json "${nb//.ipynb/_output.json}"
                      set +ex

                      ls -ltor

                      for jf in *json; do
                         filesize=$(stat -c%s "$jf")
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
322
                    
323
324
325
326
                         if (( filesize > 50001000 )); then
                             rm -fv $jf
                         fi
                      done
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
327

328
329
330
                      for outnb in *ipynb; do
                         nbreduce $outnb 3
                      done
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
331

332
333
                      cp *_output.* *.json /tmp/output
                      ls -ltor /tmp/output
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
334

335
                ''' || { echo -e "\033[31m workflow failed!\033[0m"; exit 1; }
336
337

        docker cp $cname:/tmp/output .
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
338
        docker rm -f $cname
ODA CI Bot's avatar
ODA CI Bot committed
339

340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
        ls -ltr output
        cat output/metadata.json

        f=$NB2RUN

        fb=${f//.ipynb}

        nbinspect ${fb}.ipynb  > pars_default.json

        echo "${NBARGS_PYDICT:-{\}}" > nbargs.py
        python -c 'import ast, json;\
                   json.dump(
                    { **json.load(open("pars_default.json")), 
                      **{ k: {"value": v} for
                          k, v in ast.literal_eval(open("nbargs.py").read()).items()} }, 
                    open("pars.json", "w")
                   )'

        echo -e "\033[33mpars.json:\033[0m"
        cat pars.json

        if [ "$fb" == ${ROOT_NB//.ipynb} ]; then
            nb_suffix=""
        else
            nb_suffix="-$fb"
        fi
        
        pars_suffix=$(oda-cc format '{pars[subcases_pattern][value]}_{pars[reference_instrument][value]}_{pars[nscw][value]}scw_sf{pars[systematic_fraction][value]}_{pars[osa_version][value]}')
        these_outputs=outputs$nb_suffix/$pars_suffix

        mkdir -pv $these_outputs
        mv -fv output/* $these_outputs
        cp -fv pars.json $these_outputs
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
373
374
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
375
376
377
378
379
function build-singularity() {
#       singularity -vvv create $(SINGULARITY_IMAGE) || true
        #docker save $(IMAGE_NAME) | singularity -vvv import $(SINGULARITY_IMAGE)

    docker run -v /var/run/docker.sock:/var/run/docker.sock -v /dev/shm/singularity/:/output --privileged -t --rm quay.io/singularity/docker2singularity $IMAGE
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
380
381
    mkdir -pv /data/singularity/$TEST_CASE_NAME
    mv -fv /dev/shm/singularity/* /data/singularity/$TEST_CASE_NAME/${SINGULARITY_IMAGE}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
382
383
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
384
#run-cwl:
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
385
386
#    ls 
#    cwltool job.cwl --subcases_pattern="_1" --source_name="Her X-1" --osa_version='OSA11.0' --nscw=10 --ng_sig_limit=2. --systematic_fraction=0.01
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
387

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
388
function make-args() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
389
        icversion=${1:?}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
390
391
	for c in $(ls subcases/); do 
		echo $c
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
392
		python -c 'import yaml; open("args/args-'$c'.yaml", "w").write(yaml.dump({"subcases_pattern": "'$c'", "systematic_fraction": 0.03, "osa_version": "OSA11.0-'${icversion}'", "nscw": 50}))'; 
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
393
394
395
	done
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
396
397
function submit-all-scw() {
    icversion=${1:?}   
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
398
    nscwmax=${2:-5000}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
399

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
400
    oda-node version  || { # -v!!!
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
401
402
403
404
        echo -e "\033[31mfailed to connect to ODAHUB\033[0m"
        exit 1
    }

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
405
    for scw in $(cat */subcases/*/*scw*txt subcases/*/*scw*txt | shuf |  awk '/^[0-9]{12}$/' | head -n$nscwmax) ; do 
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
406
407
408
409
410
411
412
413
        echo $scw
        
        oda-node ask \
            ii_skyimage \
                -m git://ddosa/staging-1-3 \
                -m git://findic/staging-1-3-icversion \
                -m git://ddosa11/staging-1-3 \
                -m git://useresponse/staging-1-3-osa11 \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
414
                -a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
415
416
417
418
419
420
421
                -a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
        oda-node ask \
            ii_spectra_extract \
                -m git://ddosa/staging-1-3 \
                -m git://findic/staging-1-3-icversion \
                -m git://ddosa11/staging-1-3 \
                -m git://useresponse/staging-1-3-osa11 \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
422
423
424
425
426
427
428
                -a 'ddosa.ScWData(input_scwid="'$scw'.001"),ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
                -a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
                #-a 'useresponse.CompressEBins(use_factor=4)' \
    done
}

function submit-all-cases() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
429
430
    icversion=${1:?icversion}   
    nmax=${2:?nmax}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452

    oda-node version -v || {
        echo -e "\033[31mfailed to connect to ODAHUB\033[0m"
        exit 1
    }

    for subcase in $(ls subcases/ | shuf -n $nmax) ; do 
        echo $subcase

        scws=$(cat subcases/$subcase/*scw*txt | awk '{printf $1".001,"}')
        
        oda-node ask \
            ISGRISpectraSum \
                -m git://ddosa/staging-1-3 \
                -m git://findic/staging-1-3-icversion \
                -m git://ddosa11/staging-1-3 \
                -m git://useresponse/staging-1-3-osa11 \
                -m git://process_isgri_spectra/staging-1-3-osa11 \
                -a 'process_isgri_spectra.ISGRISpectraSum(use_extract_all=True)' \
                -a 'ddosa.IDScWList(use_scwid_list="'$scws'".strip(",").split(","))' \
                -a 'process_isgri_spectra.ScWSpectraList(input_scwlist=ddosa.IDScWList)' \
                -a 'ddosa.ImagingConfig(use_SouFit=0,use_DoPart2=1,use_version="soufit0_p2"),ddosa.CatForSpectraFromImaging(use_minsig='${minsig:-5}')' \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
453
454
455
456
457
                -a 'ddosa.ICRoot(use_ic_root_version="'$icversion'")'
                #-a 'useresponse.CompressEBins(use_factor=4)' \
    done
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
458
459
460
function configure-reana() {
    reana-client secrets-add --env MINIO_KEY=${MINIO_KEY:?this should be set}
    reana-client secrets-add --env MINIO_URL=${MINIO_URL:?this should be set}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
461
    reana-client secrets-add --env MINIO_USER=${USER}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
462
463
464
    reana-client secrets-list
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
465
466
467
468
function download-cwl() {
    set -e
    echo -e "\033[33mwill download"
    curl -L --header "PRIVATE-TOKEN: $(cat ~/.gitlab-token)" \
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
469
         "https://gitlab.astro.unige.ch/api/v4/projects/integral%2Fcc-workflows%2Fcc-${TEST_CASE_NAME}/jobs/artifacts/master/download?job=build"  > artifacts.gz
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
470
471
472
    cwl=${ROOT_NB//.ipynb/.cwl}
    unzip -o artifacts.gz $cwl
    cp -fv $cwl job.cwl
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
473
474
    echo -e "\033[32mfound job.cwl:"
    ls -l job.cwl
Volodymyr SAVCHENKO's avatar
cwl    
Volodymyr SAVCHENKO committed
475
476
477

    < job.cwl awk '/fair/ || /ipyn/ {print "job.cwl >> ", $0}'

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
478
479
480
    set +e
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
481
482
function run-reana() {
      inputs=${1:?}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
483
      cwl=${2:?}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
484

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
485
486
      if [ "$cwl" == "download" ]; then
          download-cwl
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
487
          cwl=job.cwl
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
488
      fi
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
489
490
491
492
493
494
495
496

      echo "
version: 0.3.0
inputs:
  parameters:
    input: inputs.yaml
workflow:
  type: cwl
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
497
  file: ${cwl}
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
498
499
500
501
502
outputs:
  files:
   - cwl/docker_outdir/cwl.output.json
" > reana.yaml

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
503
504
505
506
507
508
509
      cp -v $inputs inputs.yaml

      name=$( oda-cc get metadata.source_short_name unnamed )
      reana-client   create  -f reana.yaml --name $name
      reana-client   start --workflow $name
}

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
510
511
512
513
514
function update-fixed-ci-version() {
    sed -i 's@raw/.*/@raw/'$(cd ci-template-cc/; git describe --always)'/@g' .gitlab-ci.yml
}

function reana-clear-all() {
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
515
    echo
Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
516
517
518
519
520
521
522
523
524
525
526
527
528
#    $ kubectl exec -i -t deployment/reana-db -- psql -U reana
#    psql> SET search_path to __reana, public; 
#    psql> UPDATE workflow SET status='stopped' WHERE name='myanalysis' AND status='running';
#    psql> \q

#    If you need just some run numbers, you can add ... AND run_number=... to the SQL query.

#    After flipping the regular reana-client delete command will work:

#    $ reana-client delete -w myanalysis --include-all-runs --include-workspace --include-records
}


Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
529
530
$@

Volodymyr SAVCHENKO's avatar
Volodymyr SAVCHENKO committed
531