Fix the behavior of Build rupture from the USGS in case the USGS id is changed after building the rupture
#6302
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # manuals -> master latest e LTS | |
| # dev/<branch>/arch | |
| # | |
| # link a pdf e constraint coi test ok | |
| # Tree Structure | |
| # -------------- | |
| # | |
| # oq-engine | |
| # <x.y>|master | |
| # <pdf_docs> | |
| # manual | |
| # latest -> <latest x.y> | |
| # LTS -> <LTS x.y> | |
| # reference | |
| # LTS -> ../LTS/reference | |
| # master -> ../master/manual | |
| # manual | |
| # LTS -> ../LTS/manual | |
| # master -> ../master/manual | |
| # advanced | |
| # LTS -> ../LTS/advanced | |
| # master -> ../master/manual | |
| # manuals | |
| # OpenQuake Manual <x.y>.pdf -> ../oq-engine/<x.y>/PDF/OpenQuake Manual <x.y>.pdf | |
| # OpenQuake Manual latest.pdf -> OpenQuake Manual <latest_(x.y)>.pdf | |
| # OpenQuake Manual LTS.pdf -> OpenQuake Manual <LTS_(x.y)>.pdf | |
| # | |
| # .dev | |
| # oq-engine | |
| # <branch> | |
| # <pdf_docs> | |
| # reference | |
| # manual | |
| # advanced | |
| # manuals | |
| # OpenQuake Manual <branch>.pdf -> ../oq-engine/<branch>/PDF/OpenQuake Manual <branch>.pdf | |
| # | |
| --- | |
| name: Docs | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| oq-release: | |
| description: 'OQ Release' | |
| required: true | |
| default: 'dev' | |
| type: choice | |
| options: | |
| - dev | |
| - master | |
| - release | |
| - LTS | |
| push: | |
| branches: [master, 'engine-*', moster, 'ongine-*'] | |
| pull_request: | |
| jobs: | |
| docs: | |
| runs-on: ubuntu-latest | |
| env: | |
| EV_NAME: ${{ github.event_name }} | |
| # during devel was DOCS_BASE: ".oq-engine-docs-refact/" | |
| DOCS_BASE: "" | |
| GITHUB_REF: ${{ github.ref }} | |
| GITHUB_HD_REF: ${{ github.head_ref }} | |
| GITHUB_BS_REF: ${{ github.base_ref }} | |
| steps: | |
| - name: run if workflow_dispatch | |
| if: github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| run: echo "Action triggered by 'workflow_dispatch' ($EV_NAME)" | |
| - name: run if push | |
| if: github.event_name == 'push' | |
| shell: bash | |
| run: echo "Action triggered by 'push' ($EV_NAME)" | |
| - name: identify github reference | |
| run: | | |
| echo "DOCS_BASE: [$DOCS_BASE]" | |
| echo "REF: $GITHUB_REF" | |
| echo "REF_NAME: $GITHUB_REF_NAME" | |
| echo "HD_REF: $GITHUB_HD_REF" | |
| echo "BS_REF: $GITHUB_BS_REF" | |
| - name: Check out the codebase | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Upgrade pip and install requirements | |
| run: | | |
| pip install -U pip | |
| pip install -r ./requirements-py311-linux64.txt | |
| pip install -r ./doc/requirements.txt | |
| - name: Install oq engine | |
| run: | | |
| pip install -e .[dev] | |
| python3 -c "import openquake.baselib as m; print(m.__version__.rpartition('.')[0])" | |
| - name: Check consistency between debian/changelog and CONTRIBUTORS.txt | |
| shell: bash | |
| run: | | |
| ./helpers/changelog2ghrel.sh --check | |
| - name: Make docs and rsync to docs.openquake.org | |
| shell: bash | |
| env: | |
| # Used | |
| BUILD: ${{ github.event.inputs.oq-release }} | |
| DOCS_SSH: ${{ secrets.DOCS_ARTIFACTS }} | |
| # GITHUB_PULL_REQUEST: ${{ github.event.number }} | |
| GITHUB_DEF_BR: ${{ github.event.repository.default_branch }} | |
| GITHUB_REF: ${{ github.ref }} | |
| GITHUB_HD_REF: ${{ github.head_ref }} | |
| GITHUB_BS_REF: ${{ github.base_ref }} | |
| UPLOAD: ${{ github.event.inputs.git-ref }} | |
| DJANGO_SETTINGS_MODULE: "openquake.server.settings" | |
| # JUST TO TEST MULTI_DOCS | |
| MULTI_DOCS_FOR_DEVEL: "false" | |
| run: | | |
| bash --version | |
| LTS_VER=$(grep '^The current LTS' README.md | sed 's/.*OpenQuake Engine //g;s/\*\*.*//g') | |
| ENG_VER=$(python3 -c "import openquake.baselib as m; print(m.__version__.rpartition('.')[0])") | |
| if [ -z "$BUILD" ]; then | |
| if [[ "$GITHUB_REF_NAME" == "master" || "$GITHUB_REF_NAME" == "moster" ]]; then | |
| BUILD=master | |
| elif [[ "$GITHUB_REF_NAME" =~ ^engine- || "$GITHUB_REF_NAME" =~ ^ongine- ]]; then | |
| BRANCH_VER="${GITHUB_REF_NAME#engine-}" | |
| if [ "$BRANCH_VER" == "$GITHUB_REF_NAME" ]; then | |
| BRANCH_VER="${GITHUB_REF_NAME#ongine-}" | |
| fi | |
| # TEST: ENABLE IF NEEDED | |
| # ENG_VER=$BRANCH_VER | |
| if [ "$BRANCH_VER" != "$ENG_VER" ]; then | |
| echo "Versions don't match (BRANCH_VER: [$BRANCH_VER] ENG_VER: [$ENG_VER])" | |
| exit 2 | |
| fi | |
| if [[ "$BRANCH_VER" == "$LTS_VER" ]]; then | |
| BUILD=LTS | |
| else | |
| BUILD=release | |
| fi | |
| else | |
| BUILD=dev | |
| fi | |
| fi | |
| REL_VERS="$(git ls-remote --heads | grep -v '^From' | grep '/engine-' | sed 's/^.*ngine-//g' | grep -v -- '-' | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n)" | |
| LATEST_VER="$(echo "$REL_VERS" | tail -n 1)" | |
| ALL_VERS="$((git ls-remote --heads | grep -v '^From' | grep '/engine-' | sed 's/^.*ngine-//g' | grep -v -- '-' ; echo "$ENG_VER") | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | uniq | tac)" | |
| PDFDOCS="manuals/" | |
| TARGET_LINK="" | |
| PDF_VER_LINK="" | |
| if [ "$BUILD" == "release" -o "$BUILD" == "LTS" ]; then | |
| TARGET="oq-engine/$ENG_VER/" | |
| # TARGET_LINK="${DOCS_BASE}oq-engine/$BUILD" | |
| PDF_VER="${ENG_VER}" | |
| if [ "$BUILD" == "release" ]; then | |
| if [ "$ENG_VER" == "$LATEST_VER" ]; then | |
| PDF_VER_LINK="latest" | |
| fi | |
| elif [ "$BUILD" == "LTS" ]; then | |
| PDF_VER_LINK="LTS" | |
| fi | |
| elif [ "$BUILD" == "master" ]; then | |
| TARGET="oq-engine/master/" | |
| PDF_VER="(master)" | |
| elif [ "$BUILD" == "dev" ]; then | |
| DOCS_BASE="${DOCS_BASE}.dev/" | |
| if echo "$GITHUB_REF_NAME" | grep -q '[0-9]\+/merge'; then | |
| TARGET="oq-engine/$GITHUB_HD_REF/" | |
| PDF_VER="($GITHUB_HD_REF)" | |
| else | |
| TARGET="oq-engine/$GITHUB_REF_NAME/" | |
| PDF_VER="($GITHUB_REF_NAME)" | |
| fi | |
| PDFDOCS="manuals/" | |
| else | |
| echo "BUILD [$BUILD] not recognized" | |
| exit 3 | |
| fi | |
| PDFDEST=${PDFDOCS} | |
| # TEST PER PDF_VER_LINK | |
| # PDF_VER_LINK=test_ver_link | |
| echo "LATEST_VER: [$LATEST_VER]" | |
| echo "BUILD: [$BUILD]" | |
| echo "LTS_VER: [$LTS_VER]" | |
| echo "ENG_VER: [$ENG_VER]" | |
| echo "TARGET: [$TARGET]" | |
| echo "TARGET_LINK: [$TARGET_LINK]" | |
| echo "PDF_VER: [$PDF_VER]" | |
| echo "PDF_VER_LINK: [$PDF_VER_LINK]" | |
| echo "PDFDOCS [$PDFDOCS]" | |
| echo "PDFDEST [$PDFDEST]" | |
| echo "----" | |
| echo "GITHUB_PULL_REQUEST: [$GITHUB_PULL_REQUEST]" | |
| echo "GITHUB_DEF_BR: [$GITHUB_DEF_BR]" | |
| echo "GITHUB_REF: [$GITHUB_REF]" | |
| echo "GITHUB_HD_REF: [$GITHUB_HD_REF]" | |
| echo "GITHUB_BS_REF: [$GITHUB_BS_REF]" | |
| echo "UPLOAD: [$UPLOAD]" | |
| # multi-version doc switcher could be runned just from master because | |
| # is the only one known the master version (from sources) | |
| set -x | |
| if [ "$MULTI_DOCS_FOR_DEVEL" == "true" -o "$BUILD" == "master" ]; then | |
| ddown_man="[" | |
| ddown_adv="[" | |
| ddown_doc="[" | |
| for v in $ALL_VERS; do | |
| ver_maj="$(echo "$v" | sed 's/\([0-9]\+\)\..*/\1/g')" | |
| ver_min="$(echo "$v" | sed 's/[0-9]\+\.\(.*\)$/\1/g')" | |
| pfx="" | |
| sfx="" | |
| if [ "$v" = "$ENG_VER" ]; then | |
| pfx="development" | |
| master_rename="true" | |
| else | |
| master_rename="false" | |
| fi | |
| # if [ "$v" = "$LATEST_VER" ]; then | |
| # if [ "$sfx" ]; then | |
| # sfx="${sfx}, latest" | |
| # else | |
| # sfx="latest" | |
| # fi | |
| # fi | |
| if [ "$v" = "$LTS_VER" ]; then | |
| if [ "$sfx" ]; then | |
| sfx="${sfx}, LTS" | |
| else | |
| sfx="LTS" | |
| fi | |
| fi | |
| if [ "$sfx" ]; then | |
| sfx=" (${sfx})" | |
| fi | |
| # policies for the user manual version switcher menu | |
| if ( (( "$ver_maj" == 3 )) && (( "$ver_min" >= 16 )) ) || (( "$ver_maj" > 3 )); then | |
| if [ "$master_rename" == "true" ]; then | |
| ver_name="${pfx}" | |
| ver_val="master" | |
| else | |
| ver_name="${ver_maj}.${ver_min}${sfx}" | |
| ver_val="${ver_maj}.${ver_min}" | |
| fi | |
| ver_url="https://docs.openquake.org/${DOCS_BASE}oq-engine/${ver_val}/manual/" | |
| if [ "$ddown_man" != "[" ]; then | |
| ddown_man="${ddown_man}, " | |
| fi | |
| ddown_man="${ddown_man}{ \"name\": \"${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }" | |
| fi | |
| # policies for the advanced manual version switcher menu | |
| if ( (( "$ver_maj" == 3 )) && (( "$ver_min" >= 16 )) ) || (( "$ver_maj" > 3 )); then | |
| if [ "$master_rename" == "true" ]; then | |
| ver_name="${pfx}" | |
| ver_val="master" | |
| else | |
| ver_name="${ver_maj}.${ver_min}${sfx}" | |
| ver_val="${ver_maj}.${ver_min}" | |
| fi | |
| if ( (( "$ver_maj" == 3 )) && (( "$ver_min" >= 19 )) ) || (( "$ver_maj" > 3 )); then | |
| adv_name="manual" | |
| else | |
| adv_name="advanced" | |
| fi | |
| ver_url="https://docs.openquake.org/${DOCS_BASE}oq-engine/${ver_val}/${adv_name}/" | |
| if [ "$ddown_adv" != "[" ]; then | |
| ddown_adv="${ddown_adv}, " | |
| fi | |
| ddown_adv="${ddown_adv}{ \"name\": \"${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }" | |
| fi | |
| # policies for the new documentation 2023 version switcher menu | |
| if ( (( "$ver_maj" == 3 )) && (( "$ver_min" >= 19 )) ) || (( "$ver_maj" > 3 )); then | |
| if [ "$master_rename" == "true" ]; then | |
| ver_name="${pfx}" | |
| ver_val="master" | |
| else | |
| ver_name="${ver_maj}.${ver_min}${sfx}" | |
| ver_val="${ver_maj}.${ver_min}" | |
| fi | |
| ver_url="https://docs.openquake.org/${DOCS_BASE}oq-engine/${ver_val}/manual/" | |
| if [ "$ddown_doc" != "[" ]; then | |
| ddown_doc="${ddown_doc}, " | |
| fi | |
| ddown_doc="${ddown_doc}{ \"name\": \"Unified ${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }" | |
| elif ( (( "$ver_maj" == 3 )) && (( "$ver_min" >= 16 )) ) || (( "$ver_maj" > 3 )); then | |
| if [ "$master_rename" == "true" ]; then | |
| ver_name="${pfx}" | |
| ver_val="master" | |
| else | |
| ver_name="${ver_maj}.${ver_min}${sfx}" | |
| ver_val="${ver_maj}.${ver_min}" | |
| fi | |
| ver_url="https://docs.openquake.org/${DOCS_BASE}oq-engine/${ver_val}/manual/" | |
| if [ "$ddown_doc" != "[" ]; then | |
| ddown_doc="${ddown_doc}, " | |
| fi | |
| ddown_doc="${ddown_doc}{ \"name\": \"Manual ${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }" | |
| if [ "$master_rename" == "true" ]; then | |
| ver_name="${pfx}" | |
| ver_val="master" | |
| else | |
| ver_name="${ver_maj}.${ver_min}${sfx}" | |
| ver_val="${ver_maj}.${ver_min}" | |
| fi | |
| ver_url="https://docs.openquake.org/${DOCS_BASE}oq-engine/${ver_val}/advanced/" | |
| if [ "$ddown_doc" != "[" ]; then | |
| ddown_doc="${ddown_doc}, " | |
| fi | |
| ddown_doc="${ddown_doc}{ \"name\": \"Advanced ${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }" | |
| if [ "$master_rename" == "true" ]; then | |
| ver_name="${pfx}" | |
| ver_val="master" | |
| else | |
| ver_name="${ver_maj}.${ver_min}${sfx}" | |
| ver_val="${ver_maj}.${ver_min}" | |
| fi | |
| ver_url="https://docs.openquake.org/${DOCS_BASE}oq-engine/${ver_val}/reference/" | |
| if [ "$ddown_doc" != "[" ]; then | |
| ddown_doc="${ddown_doc}, " | |
| fi | |
| ddown_doc="${ddown_doc}{ \"name\": \"Reference ${ver_name}\", \"version\": \"$ver_val\", \"url\": \"${ver_url}\" }" | |
| fi | |
| done | |
| ddown_man="${ddown_man}]" | |
| echo "$ddown_man" > dot_ddown_man.json | |
| ddown_adv="${ddown_adv}]" | |
| echo "$ddown_adv" > dot_ddown_adv.json | |
| ddown_doc="${ddown_doc}]" | |
| echo "$ddown_doc" > dot_ddown_doc.json | |
| fi | |
| set +x | |
| if [ "$DOCS_SSH" ]; then | |
| gpg --version | |
| gpg --quiet --batch --yes --decrypt --passphrase="$DOCS_SSH" --output ./.deploy_rsa ./.deploy_docs.enc | |
| chmod 600 ./.deploy_rsa | |
| eval $(ssh-agent -s) && ssh-add ./.deploy_rsa | |
| else | |
| echo "DOCS_SSH env variable not found, skip deploy" | |
| fi | |
| if [ "$DOCS_SSH" ]; then | |
| echo "publish dropdown version descriptions if exists" | |
| if [ -f "dot_ddown_man.json" ]; then | |
| rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' --rsync-path="mkdir -p \"${DOCS_BASE}oq-engine\" && rsync" dot_ddown_man.json "[email protected]:${DOCS_BASE}oq-engine/.ddown_man.json" | |
| fi | |
| if [ -f "dot_ddown_adv.json" ]; then | |
| rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' --rsync-path="mkdir -p \"${DOCS_BASE}oq-engine\" && rsync" dot_ddown_adv.json "[email protected]:${DOCS_BASE}oq-engine/.ddown_adv.json" | |
| fi | |
| if [ -f "dot_ddown_doc.json" ]; then | |
| rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' --rsync-path="mkdir -p \"${DOCS_BASE}oq-engine\" && rsync" dot_ddown_doc.json "[email protected]:${DOCS_BASE}oq-engine/.ddown_doc.json" | |
| fi | |
| else | |
| echo "SKIP: publish dropdown version descriptions" | |
| fi | |
| sudo apt update; sudo apt-get install -y texlive-fonts-recommended texlive-latex-extra latexmk gpg | |
| echo "Make OpenQuake Manual (HTML and PDF)" | |
| # PAY ATTENTION: engine folder is 'doc', not 'docs' | |
| # cd doc && make html && make latexpdf | |
| cd doc && make html 2>&1 | tee full.log | |
| warn_found=n | |
| err_found=n | |
| # NOTE: without '|| true' the action would fail if there are no warnings | |
| grep WARNING < full.log > warnings.log || true | |
| grep ERROR < full.log > errors.log || true | |
| if [ $(wc -c < warnings.log) -gt 0 ]; then | |
| echo | |
| echo "=== WARNINGS FOUND ===" | |
| echo | |
| cat warnings.log | |
| echo | |
| echo | |
| echo "=== WARNINGS SUMMARY ===" | |
| cat warnings.log | sed 's/.*WARNING/WARNING/g' | sed "s/WARNING: 'myst' cross-reference target not found.*/WARNING: 'myst' cross-reference target not found/g" | sed 's/WARNING: undefined label.*/WARNING: undefined label/g' | sed 's/WARNING: duplicate label.*/WARNING: duplicate label/g' | sed 's/WARNING: Duplicate explicit target name.*/WARNING: Duplicate explicit target name/g' | sort | uniq -c | |
| warn_found=y | |
| else | |
| echo "=== NO WARNINGS FOUND ===" | |
| fi | |
| if [ $(wc -c < errors.log) -gt 0 ]; then | |
| echo | |
| echo "=== ERRORS FOUND ===" | |
| echo | |
| cat errors.log | |
| echo | |
| err_found=y | |
| else | |
| echo "=== NO ERRORS FOUND ===" | |
| fi | |
| if [ "$warn_found" = "y" -o "$err_found" = "y" ]; then | |
| echo "fix all warnings and errors to complete this workflow" | |
| exit 1 | |
| fi | |
| # if [ "$DOCS_SSH" ]; then | |
| # echo "Uploading OpenQuake Manual (PDF)" | |
| # rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --rsync-path="mkdir -p \"${DOCS_BASE}${TARGET}PDF\" && rsync" build/latex/OpenQuakeEngineManual.pdf "[email protected]:${DOCS_BASE}${TARGET}PDF/OpenQuake Manual ${PDF_VER}.pdf" | |
| # ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}${PDFDOCS}\"; ln -sf \"../${TARGET}PDF/OpenQuake Manual ${PDF_VER}.pdf\" .'" | |
| # if [ "$PDF_VER_LINK" ]; then | |
| # echo "Found PDF_VER_LINK set [$PDF_VER_LINK]" | |
| # ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}manuals/\"; ln -sf \"OpenQuake Manual ${PDF_VER}.pdf\" \"OpenQuake Manual (${PDF_VER_LINK}).pdf\"'" | |
| # fi | |
| # else | |
| # echo "SKIP Uploading OpenQuake Manual (PDF)" | |
| # fi | |
| if [ "$DOCS_SSH" ]; then | |
| echo "Uploading New OpenQuake Documentation (HTML)" | |
| rsync -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --rsync-path="mkdir -p \"${DOCS_BASE}${TARGET}manual/\" && rsync" -s -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -ax --delete _build/html/ "[email protected]:/var/www/docs.openquake.org/${DOCS_BASE}${TARGET}manual/" | |
| echo "------" | |
| else | |
| echo "SKIP Uploading New OpenQuake Documentation (HTML)" | |
| fi | |
| if [ "$BUILD" == "LTS" ]; then | |
| if [ "$DOCS_SSH" ]; then | |
| ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}oq-engine/\"; ln -snf \"$ENG_VER\" LTS'" | |
| else | |
| echo "SKIP create LTS link" | |
| fi | |
| fi | |
| if [ "$BUILD" == "release" -o "$BUILD" == "LTS" ]; then | |
| if [ "$LATEST_VER" == "$ENG_VER" ]; then | |
| if [ "$DOCS_SSH" ]; then | |
| ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] "bash -cx 'cd \"${DOCS_BASE}oq-engine/\"; ln -snf \"$ENG_VER\" latest'" | |
| else | |
| echo "SKIP create latest link" | |
| fi | |
| fi | |
| fi |