Merge branch 'candidate-8.0.x'

Signed-off-by: Richard Chapman <[email protected]>

.github/workflows/smoketest.yml

@@ -302,7 +302,7 @@ jobs:
         find -name '.git*' -prune -o -newer ../cmake-timestamp -type f -print | cpio -p -dum ../src-cache
         echo "Merge patch files:"
         git diff --name-only ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }}
-        git diff ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} > ../merge-patch
+        git diff --binary ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} > ../merge-patch
 
     # - alter stock ecl-test.json, to point to install dir (using jq)
     # - create a clean copy of the install directory (for later reuse)

CMakeLists.txt

@@ -69,6 +69,11 @@
 
 project(hpccsystems-platform)
 
+# Stupid workaround. See https://gitlab.kitware.com/cmake/cmake/-/issues/21378
+# If cmake runs twice, cmake might not correctly set the compiler version variables
+unset ( ENV{CC} )
+unset ( ENV{CXX} )
+
 cmake_minimum_required(VERSION 3.13.0)
 
 set(TOP_LEVEL_PROJECT ON)

cmake_modules/dependencies/hirsute.cmake

@@ -0,0 +1,6 @@
+# Ubuntu 21.04
+SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS g++ openssh-client openssh-server expect rsync libapr1 python3 psmisc curl )
+
+if(SPARK)
+    SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS "openjdk-11-jdk" )
+endif(SPARK)

cmake_modules/getpackagerevisionarch.sh

@@ -83,7 +83,7 @@ elif [ -e /etc/redhat-release ]; then
   if [ -x /bin/rpm ]; then
     OUTPUT="${OUTPUT}\npackage=rpm"
     OS_GROUP=`/bin/rpm -q --qf "%{NAME}" --whatprovides /etc/redhat-release | sed 's/-release.*//' |  tr '[A-Z]' '[a-z]'`
-    REDHAT_VERSION=`/bin/rpm -q --qf "%{VERSION}" --whatprovides /etc/redhat-release`
+    REDHAT_VERSION=`/bin/rpm -q --qf "%{VERSION}" --whatprovides /etc/redhat-release | cut -f1 -d"."`
     case "$OS_GROUP" in
       "centos"* | "fedora")
         if [ ${NOARCH} -eq 0 ]; then

cmake_modules/go_gold.sh

@@ -72,7 +72,7 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
   # but only copy helm chart sources across for "latest stable" version
 
   HPCC_DIR="$( pwd )"
-  pushd ../helm-chart 2>&1 > /dev/null
+  doit2 "pushd ../helm-chart 2>&1 > /dev/null"
   doit "git fetch $REMOTE"
   doit "git checkout master"
   doit "git merge --ff-only $REMOTE/master"
@@ -85,9 +85,9 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
     doit "rm ./helm/hpcc/*.bak" 
     doit "git add -A ./helm"
   fi
-  cd docs
-  for f in `find ${HPCC_DIR}/helm/examples -name Chart.yaml` ; do 
-    doit "helm package ${f%/*}/"  
+  doit2 "cd docs"
+  for f in `find ${HPCC_DIR}/helm/examples ${HPCC_DIR}/helm/managed -name Chart.yaml` ; do
+    doit "helm package ${f%/*}/ --dependency-update"
   done
   doit "helm package ${HPCC_DIR}/helm/hpcc/"
   doit "helm repo index . --url https://hpcc-systems.github.io/helm-chart"
@@ -98,5 +98,5 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
     doit "git tag $FORCE $HPCC_MAJOR.$HPCC_MINOR.$HPCC_POINT && git push $REMOTE $HPCC_MAJOR.$HPCC_MINOR.$HPCC_POINT $FORCE"
   fi
   doit "git push $REMOTE master $FORCE"
-  popd
+  doit2 "popd 2>&1 > /dev/null"
 fi

cmake_modules/go_rc.sh

@@ -89,19 +89,19 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
   # We publish any tagged version of helm chart to the helm-chart repo
   # but only copy helm chart sources across for "latest stable" version
   HPCC_DIR="$( pwd )"
-  pushd ../helm-chart 2>&1 > /dev/null
+  doit2 "pushd ../helm-chart 2>&1 > /dev/null"
   doit "git fetch $REMOTE"
   doit "git checkout master"
   doit "git merge --ff-only $REMOTE/master"
   doit "git submodule update --init --recursive"
   HPCC_PROJECTS=hpcc-helm
   HPCC_NAME=HPCC
-  cd docs
+  doit2 "cd docs"
   doit "helm package ${HPCC_DIR}/helm/hpcc/"
   doit "helm repo index . --url https://hpcc-systems.github.io/helm-chart"
   doit "git add *.tgz"
 
   doit "git commit -a -s -m \"$HPCC_NAME Helm Charts $HPCC_SHORT_TAG Release Candidate $HPCC_SEQUENCE\""
   doit "git push $REMOTE master $FORCE"
-  popd
+  doit2 "popd 2>&1 > /dev/null"
 fi

cmake_modules/parse_cmake.sh

@@ -110,6 +110,12 @@ function doit()
     if [ -z "$DRYRUN" ] ; then eval $1 ; fi
 }
 
+function doit2()
+{
+    if [ -n "$VERBOSE" ] || [ -n "$DRYRUN" ] ; then echo $1 ; fi
+    eval $1
+}
+
 function set_tag()
 {
     local _prefix=$1

dali/base/dafdesc.cpp

@@ -2387,11 +2387,19 @@ static const char * defaultWindowsBaseDirectories[__grp_size][MAX_REPLICATION_LE
     };
 static const char * defaultUnixBaseDirectories[__grp_size][MAX_REPLICATION_LEVELS] =
     {
+#ifdef _CONTAINERIZED
+        { "/var/lib/HPCCSystems/hpcc-data", "/var/lib/HPCCSystems/hpcc-mirror" },
+        { "/var/lib/HPCCSystems/hpcc-data", "/var/lib/HPCCSystems/hpcc-mirror" },
+        { "/var/lib/HPCCSystems/hpcc-data", "/var/lib/HPCCSystems/hpcc-data2", "/var/lib/HPCCSystems/hpcc-data3", "/var/lib/HPCCSystems/hpcc-data4" },
+        { "/var/lib/HPCCSystems/hpcc-data", "/var/lib/HPCCSystems/hpcc-mirror" },
+        { "/var/lib/HPCCSystems/hpcc-data", "/var/lib/HPCCSystems/hpcc-mirror" },
+#else
         { "/var/lib/HPCCSystems/hpcc-data/thor", "/var/lib/HPCCSystems/hpcc-mirror/thor" },
         { "/var/lib/HPCCSystems/hpcc-data/thor", "/var/lib/HPCCSystems/hpcc-mirror/thor" },
         { "/var/lib/HPCCSystems/hpcc-data/roxie", "/var/lib/HPCCSystems/hpcc-data2/roxie", "/var/lib/HPCCSystems/hpcc-data3/roxie", "/var/lib/HPCCSystems/hpcc-data4/roxie" },
         { "/var/lib/HPCCSystems/hpcc-data/eclagent", "/var/lib/HPCCSystems/hpcc-mirror/eclagent" },
         { "/var/lib/HPCCSystems/hpcc-data/unknown", "/var/lib/HPCCSystems/hpcc-mirror/unknown" },
+#endif
     };
 static const char *componentNames[__grp_size] =
     {
@@ -3199,45 +3207,38 @@ bool GroupInformation::checkIsSubset(const GroupInformation & other)
 void GroupInformation::createStoragePlane(IPropertyTree * storage, unsigned copy) const
 {
     IPropertyTree * plane = storage->addPropTree("planes");
-    if (copy == 0)
-    {
-        plane->setProp("@name", name);
-    }
-    else
-    {
-        StringBuffer mirrorname;
-        mirrorname.append(name).append("_mirror").append(copy);
-        plane->setProp("@name", mirrorname);
-    }
+    StringBuffer mirrorname;
+    const char * planeName = name;
+    if (copy != 0)
+        planeName = mirrorname.append(name).append("_mirror").append(copy);
+
+    plane->setProp("@name", planeName);
 
     //URL style drop zones don't generate a host entry, and will have a single device
     if (ordinality() != 0)
     {
-        if (container)
+        if (container && (copy == 0))
         {
-            plane->setProp("@hosts", container->name);
-            if (containerOffset)
-                plane->setPropInt("@start", containerOffset);
-            if (ordinality() != container->ordinality())
-                plane->setPropInt("@size", ordinality());
+            //hosts will be expanded by normalizeHostGroups
+            plane->setProp("@hostGroup", container->name);
         }
         else
-            plane->setProp("@hosts", name);
+        {
+            //Host group has been created that matches the name of the storage plane
+            plane->setProp("@hostGroup", planeName);
+        }
 
         if (ordinality() > 1)
             plane->setPropInt("@numDevices", ordinality());
     }
 
-    if (copy)
-        plane->setPropInt("@offset", copy);
-
     if (dir.length())
         plane->setProp("@prefix", dir);
     else
         plane->setProp("@prefix", queryBaseDirectory(groupType, copy));
 
-    if (dropZone)
-        plane->setPropBool("@dropZone", true);
+    const char * label = dropZone ? "lz" : "data";
+    addPTreeItem(plane, "label", label);
 
     //MORE: If container is identical to this except for the name we could generate an information tag @alias
 }
@@ -3283,6 +3284,9 @@ static void generateHosts(IPropertyTree * storage, GroupInfoArray & groups)
     ForEachItemIn(i, groups)
     {
         GroupInformation & cur = groups.item(i);
+        if (cur.ordinality() == 0)
+            continue;
+
         GroupInformation * container = nullptr;
         for (unsigned j=0; j < i; j++)
         {
@@ -3292,16 +3296,34 @@ static void generateHosts(IPropertyTree * storage, GroupInfoArray & groups)
         }
 
         // No containing hostGroup found, so generate a new entry for this set of hosts
-        if (!cur.container && cur.ordinality())
+        if (!cur.container)
         {
             IPropertyTree * plane = storage->addPropTree("hostGroups");
             plane->setProp("@name", cur.name);
             StringBuffer host;
             ForEachItemIn(i, cur.hosts)
-            {
-                IPropertyTree * entry = plane->addPropTreeArrayItem("hosts", createPTree());
-                entry->setProp("", cur.hosts.item(i));
-            }
+                addPTreeItem(plane, "hosts", cur.hosts.item(i));
+        }
+        else if (cur.containerOffset || cur.ordinality() != cur.container->ordinality())
+        {
+            IPropertyTree * plane = storage->addPropTree("hostGroups");
+            plane->setProp("@name", cur.name);
+            plane->setProp("@hostGroup", cur.container->name);
+            if (cur.containerOffset)
+                plane->setPropInt("@offset", cur.containerOffset);
+            if (cur.ordinality() != cur.container->ordinality())
+                plane->setPropInt("@count", cur.ordinality());
+            cur.container = nullptr;
+        }
+
+        if (cur.groupType == grp_thor)
+        {
+            VStringBuffer mirrorName("%s_mirror", cur.name.str());
+
+            IPropertyTree * plane = storage->addPropTree("hostGroups");
+            plane->setProp("@name", mirrorName);
+            plane->setProp("@hostGroup", cur.name);
+            plane->setPropInt("@delta", 1);
         }
     }
 }
@@ -3326,9 +3348,6 @@ static void doInitializeStorageGroups(bool createPlanesFromGroups)
     if (!storage)
         storage.set(globalConfig->addPropTree("storage"));
 
-    //Ensure that host groups that are defined in terms of other host groups are expanded out so they have an explicit list of hosts
-    normalizeHostGroups();
-
 #ifndef _CONTAINERIZED
     if (createPlanesFromGroups && !storage->hasProp("planes"))
     {
@@ -3372,7 +3391,7 @@ static void doInitializeStorageGroups(bool createPlanesFromGroups)
 
                     next->dir.set(cur.queryProp("@directory"));
                     next->dropZone= true;
-                    if (ip)
+                    if (ip && !strieq(ip, "localhost"))
                         next->hosts.append(ip);
                     appendGroup(allGroups, next.getClear());
                 }
@@ -3399,6 +3418,8 @@ static void doInitializeStorageGroups(bool createPlanesFromGroups)
     }
 #endif
 
+
+    //Ensure that host groups that are defined in terms of other host groups are expanded out so they have an explicit list of hosts
     normalizeHostGroups();
 
     //Groups are case insensitve, so add an extra key to the storage items to allow them to be

dali/base/dameta.cpp

@@ -75,7 +75,7 @@ void normalizeHostGroups()
             for (unsigned i=0; i < numHosts; i++)
             {
                 unsigned baseIndex = offset + (i + delta) % numHosts;
-                cur.addPropTree("hosts")->setProp(nullptr, hosts.item(baseIndex));
+                addPTreeItem(&cur, "hosts", hosts.item(baseIndex));
             }
         }
     }

dockerfiles/platform-core-debug/Dockerfile

@@ -28,7 +28,7 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en  
 ENV LC_ALL en_US.UTF-8     
 
-RUN apt-get install -y libcap2-bin zip
+RUN apt-get install -y libcap2-bin zip libarchive13
 RUN setcap cap_sys_ptrace+eip /usr/bin/gdb
 
 ENV PATH="/opt/HPCCSystems/bin:${PATH}"

dockerfiles/platform-core/Dockerfile

@@ -45,6 +45,7 @@ RUN apt-get install -y \
   psmisc \
   r-base-core \
   zip \
+  libarchive13 \
   curl
 
 ARG USE_CPPUNIT=1

docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml

@@ -1616,6 +1616,8 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           </orderedlist>
         </sect3>
 
+        <para />
+
         <?hard-pagebreak ?>
 
         <sect3>
@@ -1692,6 +1694,63 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
 
         <?hard-pagebreak ?>
 
+        <para>
+          <emphasis role="bold">To add feature access flags to a preexisting
+          ECL Watch configuration:</emphasis>
+        </para>
+
+        <para>If you update the platform, but are using a preexisting
+        configuration, you could encounter a situation where Feature level
+        access flags are not automatically created. Missing flags can deny
+        access to users trying to access features in the system. </para>
+
+        <orderedlist>
+          <listitem>
+            <para>In Configuration Manager, open your working copy of the
+            environment.xml and enable <emphasis>Write
+            Access.</emphasis></para>
+          </listitem>
+
+          <listitem>
+            <para>On the left, select the ESP that is hosting the ECL Watch
+            service.</para>
+          </listitem>
+
+          <listitem>
+            <para>On the right, select the <emphasis>ESP Service
+            Bindings</emphasis> tab.</para>
+
+            <para>Manually add the access flag for the new service in the
+            Feature Authentication table.</para>
+          </listitem>
+
+          <listitem>
+            <para>Right-click on the Feature Authentication table, then select
+            <emphasis>Add</emphasis></para>
+
+            <para>
+              <graphic fileref="images/CM-img08AddFlag.jpg"
+                       vendor="configmgrSS" />
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>Provide the values for path and resource (usually the same
+            value, for example, WsStoreAccess).</para>
+          </listitem>
+
+          <listitem>
+            <para>Click the disk icon to save</para>
+          </listitem>
+
+          <listitem>
+            <para>Copy the file to every node, then restart the system.</para>
+
+            <para>The OUs should get created in LDAP automatically upon
+            restart.</para>
+          </listitem>
+        </orderedlist>
+
         <sect3>
           <title>Esp - myesp AuthenticationTab</title>
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3950,8 +3950,10 @@ unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHq
 
         bool isComplex = extraFilter || !extractor.isSingleMatchCondition();
         const char * baseClass = isComplex ? "RtlComplexIfBlockTypeInfo" : "RtlSimpleIfBlockTypeInfo";
-        if (isComplex)
-            fieldType |= RFTMnoserialize;
+        //Currently interpreted ifblocks() are far too inefficient to be used remotely, so unconditionally mark it as noserialize
+        //change when HPCC-25794 is fixed
+        //if (isComplex)
+        fieldType |= RFTMnoserialize;
 
         //The ifblock needs a unique instance of the class to evaluate the test
         BuildCtx fieldclassctx(declarectx);

ecllibrary/std/DataPatterns/BestRecordStructure.ecl

@@ -47,7 +47,6 @@ EXPORT BestRecordStructure(inFile, sampling = 100, emitTransform = FALSE, textOu
     #UNIQUENAME(bestNamePrefix);
     #UNIQUENAME(recLevel);
 
-    IMPORT DataPatterns;
     IMPORT Std;
 
     // Attribute naming note:  In order to reduce symbol collisions with calling
@@ -216,7 +215,7 @@ EXPORT BestRecordStructure(inFile, sampling = 100, emitTransform = FALSE, textOu
         );
 
     // Get the best data types from the Profile() function
-    LOCAL __patternRes := DataPatterns.Profile(inFile, features := 'best_ecl_types', sampleSize := sampling);
+    LOCAL __patternRes := Std.DataPatterns.Profile(inFile, features := 'best_ecl_types', sampleSize := sampling);
 
     // Append the derived 'best' data types to the field information we
     // already collected