From 255d362d346ca1bd5ab07016859b444605a6de78 Mon Sep 17 00:00:00 2001 From: Dataliberate Date: Tue, 16 Jul 2019 17:16:15 +0100 Subject: [PATCH] Updated buildreleasefiles.sh to build all release files Updated RELEASING.TXT to reflect changes --- RELEASING.TXT | 34 ++-------------- scripts/buildreleasefiles.sh | 77 +++++++++++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/RELEASING.TXT b/RELEASING.TXT index 48f36202f1..88bf166508 100644 --- a/RELEASING.TXT +++ b/RELEASING.TXT @@ -46,46 +46,18 @@ e.g. data/releases/2.1/schema.nt data/releases/2.1/schema.rdfa -Of these, +All these are all created and coppied where appropriate by running the 'scripts/buildreleasefiles.sh' script +[since version 3.9] - * schema.rdfa is a frozen one-time copy of data/schema.rdfa - * schema-all.html is generated from this via running the Web app and - capturing via curl of /version/build-latest, - e.g. http://localhost:8080/version/build-latest - when served locally via "dev_appserver ." - For now, you need to edit this html to remove webschemas.org from heading section. - - * To create .nt, .nq, etc files use scripts/buildreleasefiles.sh - - * README.md should be copied/updated accordingly. TODO: simplify this file. Example usage: + scripts/buildreleasefiles.sh 3.9 -First run scripts/buildTermConfig.sh - ensure all config files are referenced. - -Next copy snapshot into place: - -* cp data/schema.rdfa data/releases/2.2/schema.rdfa - -* Now the HTML snapshot itself. Assuming a local server (e.g. from 'dev_appserver .') we can do: - * curl http://localhost:8080/version/build-latest > data/releases/3.2/schema-all.html (taking care with the version number, of course) - or this, to use webschemas: curl https://webschemas.org/version/build-latest > data/releases/3.4/schema-all.html - * clean up the html including the bit that says you're in a developer view etc. - -Then generate N-Triples etc. versions: -* Ensure that the EXTENSIONS variable at the top of the script 'scripts/buildreleasefiles.sh' contains all the extensions for this release. -* scripts/buildreleasefiles.sh 3.1 - Check listed extensions is correct update if not - -* Run './scripts/buildsitemap.py' make sure sitemap is up to date - -* Again run scripts/buildTermConfig.sh - ensure all config & docs files are referenced * Inspect snapshot, add to git and push to repo. * ls -l data/releases/2.2/ * git add data/releases/2.2/* * git push - * update sites: e.g. ./scripts/updateAppEngine.sh to update sdo-phobos.appspot.com, webschemas.org - * Inspect /version/{x.y}/ on those sites manually; check for newly added terms etc. * TODO: more QA and automation would be highly appropriate here. diff --git a/scripts/buildreleasefiles.sh b/scripts/buildreleasefiles.sh index 08cf3c9f09..a4fd0ca3b8 100755 --- a/scripts/buildreleasefiles.sh +++ b/scripts/buildreleasefiles.sh @@ -12,11 +12,13 @@ then fi function usage { - echo "usage: $(basename $0) [-y] [-e] [-c] [-o] [-limit somevalue] VERSION" + echo "usage: $(basename $0) [-y] [-e] [-c] [-o] [s] [m] [-limit somevalue] VERSION" echo " -y Assume yes to continue" echo " -e No extentstions (only produce core and all-layers)" echo " -c No context file" echo " -o No owl file" + echo " -s No schema-all.htmlfile" + echo " -m No sitemap file" echo " -l \"Output types\" (json-ld|turtle|nt|nquads|rdf|csv)" } @@ -29,9 +31,11 @@ fi LIMIT="" AUTORUN=0 CONTEXT=1 +RELS=1 OWL=1 +MAP=1 EXTS=1 -while getopts 'yecol:' OPTION; do +while getopts 'yecsoml:' OPTION; do case "$OPTION" in y) AUTORUN=1 @@ -42,6 +46,12 @@ while getopts 'yecol:' OPTION; do CONTEXT=0 ;; + s) + RELS=0 + ;; + m) + MAP=0 + ;; o) OWL=0 ;; @@ -69,9 +79,12 @@ VER=$1 DIR="./data/releases/$1" +echo +echo "$PROG:About to build release files for version $VER" +echo " Including extensions: $EXTENSIONS" + if [ $AUTORUN -eq 0 ] then - echo "$PROG:\n\tAbout to build release files for version $VER \n\tIncluding extensions: $EXTENSIONS" read -r -p "Continue? y/n: " response case $response in [yY]) @@ -83,6 +96,12 @@ then esac fi +echo +echo -n "Preparing by running buildTermConfig.sh... " +./scripts/buildTermConfig.sh +echo " Prepared." +sleep 3 + if [ ! -d $DIR ] then echo "Creating $VER directory" @@ -93,15 +112,28 @@ fi if [ ! -d $DIR ] then echo "Failed to create $DIR! Aborting" + exit 1 fi - -echo "Cleaning directory" +sleep 1 +echo -n "Cleaning directory... " rm -f $DIR/*.jsonld 2>&1 > /dev/null rm -f $DIR/*.rdf 2>&1 > /dev/null rm -f $DIR/*.nq 2>&1 > /dev/null rm -f $DIR/*.nt 2>&1 > /dev/null rm -f $DIR/*.ttl 2>&1 > /dev/null rm -f $DIR/*.csv 2>&1 > /dev/null +rm -f $DIR/schema.rdfa 2>&1 > /dev/null +rm -f $DIR/README.md 2>&1 > /dev/null +rm -f $DIR/schema-all.html 2>&1 > /dev/null +rm -f $DIR/schemaorg.owl 2>&1 > /dev/null +echo " cleaned." +sleep 2 + +echo -n "Copying schema.rdfa and README.md into release directory... " +cp ./data/schema.rdfa $DIR +cp ./README.md $DIR +echo " copied" +sleep 2 function dump { in=$1 @@ -134,6 +166,9 @@ function dump { ./scripts/exportgraphs.py -i "$in" -e "$ex1" -e "$ex2" -g "#$VER" $forms -o $DIR/$file 2>&1 > /dev/null } +echo +echo "Creating dump files - this takes a while......." +sleep 2 echo echo "Creating core: " dump core extensions schema "$LIMIT" @@ -157,14 +192,44 @@ if [ $CONTEXT -eq 1 ] then echo "Creating archive context file" ./scripts/buildarchivecontext.py -o schemaorgcontext.jsonld -d $DIR + echo + echo "Created archive context file" + echo + sleep 2 fi +if [ $RELS -eq 1 ] +then + echo "creating schema-all.html file" + ./scripts/buildreleasespage.py -o $DIR/schema-all.html + echo + echo "created schema-all.html file" + echo + sleep 2 +fi if [ $OWL -eq 1 ] then echo "creating owl file" ./scripts/buildowlfile.py + echo + echo "created owl file" + echo + echo -n "Copying owl file to $DIR ... " + cp ./docs/schemaorg.owl $DIR + echo " copied." + sleep 2 +fi +if [ $RELS -eq 1 ] +then + echo + echo "Creating sitemap file" + ./scripts/buildsitemap.py + echo + echo "created sitemap file" + echo + sleep 2 fi -echo done +echo "Done!"