Upgrade openssl, python-cryptography and add support for rpmbuild macros

Change-Id: Idb4939111f1ee1dc8cdfe3b2ce2fd7f1ebb3cd32 Reviewed-on: http://photon-jenkins.eng.vmware.com/619 Tested-by: jenkins-photon <[email protected]> Reviewed-by: Divya Thaluru <[email protected]>
qiyanqi · Mar 10, 2016 · 90d8aca · 90d8aca
1 parent eb0e865
commit 90d8aca
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 36 deletions.
diff --git a/Makefile b/Makefile
@@ -216,7 +216,8 @@ packages: check $(PHOTON_STAGE) $(PHOTON_PUBLISH_RPMS) $(PHOTON_SOURCES) $(CONTA
                 -d $(PHOTON_DIST_TAG) \
                 -n $(PHOTON_BUILD_NUMBER) \
                 -v $(PHOTON_RELEASE_VERSION) \
-                -w $(PHOTON_DATA_DIR)/pkg_info.json\
+                -w $(PHOTON_DATA_DIR)/pkg_info.json \
+                -g $(PHOTON_DATA_DIR)/pkg_build_options.json \
                 $(PHOTON_RPMCHECK_OPTION) \
                 -t ${THREADS}
 
@@ -485,6 +486,7 @@ check-packer-ovf-plugin:
                               -d $(PHOTON_DIST_TAG) \
                               -n $(PHOTON_BUILD_NUMBER) \
                               -v $(PHOTON_RELEASE_VERSION) \
+                              -g $(PHOTON_DATA_DIR)/pkg_build_options.json \
                               $(PHOTON_RPMCHECK_OPTION) \
                               -l $(PHOTON_LOGS_DIR)
 

diff --git a/SPECS/openssl/openssl.spec b/SPECS/openssl/openssl.spec
@@ -1,14 +1,14 @@
 Summary:	Management tools and libraries relating to cryptography
 Name:		openssl
-Version:	1.0.2f
+Version:	1.0.2g
 Release:	1%{?dist}
 License:	OpenSSL
 URL:		http://www.openssl.org
 Group:		System Environment/Security
 Vendor:		VMware, Inc.
 Distribution:	Photon
 Source0:	http://www.openssl.org/source/%{name}-%{version}.tar.gz
-%define sha1 openssl=2047c592a6e5a42bd37970bdb4a931428110a927
+%define sha1 openssl=36af23887402a5ea4ebef91df8e61654906f58f2
 Patch0:		c_rehash.patch
 Patch1:     openssl-1.0.2f-ipv6apps.patch
 Requires:	bash glibc libgcc 
@@ -58,6 +58,7 @@ export CFLAGS="%{optflags}"
 	--openssldir=/%{_sysconfdir}/ssl \
 	shared \
 	zlib-dynamic \
+        %{?_with_fips} \
 	-Wa,--noexecstack "${CFLAGS}" "${LDFLAGS}"
 # does not support -j yet
 make
@@ -107,6 +108,8 @@ rm -rf %{buildroot}/*
 /%{_bindir}/c_rehash
 
 %changelog
+*   Mon Mar 07 2016 Anish Swaminathan <[email protected]> 1.0.2g-1
+-   Upgrade to 1.0.2g
 *   Wed Feb 03 2016 Xiaolin Li <[email protected]> 1.0.2f-1
 -   Update to version 1.0.2f
 *   Mon Feb 01 2016 Anish Swaminathan <[email protected]> 1.0.2e-3

diff --git a/SPECS/python-cryptography/python-cryptography.spec b/SPECS/python-cryptography/python-cryptography.spec
@@ -1,13 +1,12 @@
 Summary:        Python cryptography library
 Name:           python-cryptography
-Version:        1.2.2
+Version:        1.2.3
 Release:        1%{?dist}
 Url:            https://cryptography.io
 License:        ASL 2.0
 Group:          Development/Languages/Python
 Source0:        https://pypi.python.org/packages/source/c/cryptography/cryptography-%{version}.tar.gz
-%define sha1 cryptography=16088fb22c2ea4cd445bcfca6a6bbb99b79dc285
-
+%define sha1 cryptography=a8a8083e70875423bd72899ca99890b788189205
 BuildRequires: python2
 BuildRequires: python2-libs
 BuildRequires: python2-devel
@@ -38,7 +37,9 @@ python setup.py install --prefix=%{_prefix} --root=%{buildroot}
 %{python_sitelib}/*
 
 %changelog
-*	Thu Feb 26 2016 Anish Swaminathan <[email protected]> 1.2.2-1
+*	Mon Mar 07 2016 Anish Swaminathan <[email protected]> 1.2.3-1
+-	Upgrade to 1.2.3
+*	Fri Feb 26 2016 Anish Swaminathan <[email protected]> 1.2.2-1
 -	Upgrade version to 1.2.2
 *	Thu Jan 21 2016 Anish Swaminathan <[email protected]> 1.2.1-1
 -	Upgrade version

diff --git a/common/data/pkg_build_options.json b/common/data/pkg_build_options.json
@@ -0,0 +1,6 @@
+{
+    "dummy_package": { 
+        "files": ["path_to_file1_to_be_copied","path_to_file2_to_be_copied"], 
+        "macros": ["macro1 value1","macro2 value2"]
+    }
+}
diff --git a/support/package-builder/PackageBuilder.py b/support/package-builder/PackageBuilder.py
@@ -9,7 +9,7 @@
 
 class PackageBuilder(object):
 
-    def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,logName=None,logPath=None):
+    def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,listBuildOptionPackages,pkgBuildOptionFile,logName=None,logPath=None):
         if logName is None:
             logName = "PackageBuilder"
         if logPath is None:
@@ -20,6 +20,8 @@ def __init__(self,mapPackageToCycles,listAvailableCyclicPackages,logName=None,lo
         self.mapPackageToCycles = mapPackageToCycles
         self.listAvailableCyclicPackages = listAvailableCyclicPackages
         self.listNodepsPackages = ["glibc","gmp","zlib","file","binutils","mpfr","mpc","gcc","ncurses","util-linux","groff","perl","texinfo","rpm","openssl","go"]
+        self.listBuildOptionPackages=listBuildOptionPackages
+        self.pkgBuildOptionFile=pkgBuildOptionFile
 
     def prepareBuildRoot(self,chrootName,isToolChainPackage=False):
         chrootID=None
@@ -100,7 +102,7 @@ def buildPackage(self,package):
                 pkgUtils.installRPMSInAOneShot(chrootID,destLogPath)
                 self.logger.info("Finished installing the build time dependent packages......")
             pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
-            pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath)
+            pkgUtils.buildRPMSForGivenPackage(package,chrootID,self.listBuildOptionPackages,self.pkgBuildOptionFile,destLogPath)
             self.logger.info("Successfully built the package:"+package)
         except Exception as e:
             self.logger.error("Failed while building package:" + package)

diff --git a/support/package-builder/PackageManager.py b/support/package-builder/PackageManager.py
@@ -26,6 +26,8 @@ def __init__(self,logName=None,logPath=None):
         self.mapOutputThread={}
         self.mapThreadsLaunchTime={}
         self.listAvailableCyclicPackages=[]
+        self.listBuildOptionPackages=[]
+        self.pkgBuildOptionFile="" 
 
     def readPackageBuildData(self, listPackages):
         try:
@@ -87,24 +89,28 @@ def calculateParams(self,listPackages):
     def buildToolChain(self):
         try:
             tUtils=ToolChainUtils()
-            tUtils.buildCoreToolChainPackages()
+            tUtils.buildCoreToolChainPackages(self.listBuildOptionPackages, self.pkgBuildOptionFile)
         except Exception as e:
             self.logger.error("Unable to build tool chain")
             self.logger.error(e)
             raise e
 
-    def buildToolChainPackages(self, buildThreads):
+    def buildToolChainPackages(self, listBuildOptionPackages, pkgBuildOptionFile, buildThreads):
         self.buildToolChain()
         self.buildGivenPackages(constants.listToolChainPackages, buildThreads)
 
-    def buildPackages(self,listPackages, buildThreads):
-        self.buildToolChainPackages(buildThreads)
+    def buildPackages(self,listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads):
+        self.listBuildOptionPackages = listBuildOptionPackages
+        self.pkgBuildOptionFile = pkgBuildOptionFile
+        self.buildToolChainPackages(listBuildOptionPackages, pkgBuildOptionFile, buildThreads)
         self.buildGivenPackages(listPackages, buildThreads)
 
     def initializeThreadPool(self,statusEvent):
         ThreadPool.clear()
         ThreadPool.mapPackageToCycle=self.mapPackageToCycle
         ThreadPool.listAvailableCyclicPackages=self.listAvailableCyclicPackages
+        ThreadPool.listBuildOptionPackages=self.listBuildOptionPackages
+        ThreadPool.pkgBuildOptionFile=self.pkgBuildOptionFile
         ThreadPool.logger=self.logger
         ThreadPool.statusEvent=statusEvent
 

diff --git a/support/package-builder/PackageUtils.py b/support/package-builder/PackageUtils.py
@@ -7,6 +7,8 @@
 from time import sleep
 import PullSources
 from PackageInfo import SourcePackageInfo
+import json
+import collections
 
 class PackageUtils(object):
 
@@ -132,9 +134,17 @@ def copySourcesTobuildroot(self,listSourceFiles,package,destDir):
                 self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")
                 raise Exception("Multiple sources found")
             self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0])
-            shutil.copy2(sourcePath[0],  destDir)
+            shutil.copy2(sourcePath[0], destDir)
 
-    def buildRPMSForGivenPackage(self,package, chrootID,destLogPath=None):
+    def copyAdditionalBuildFiles(self,listAdditionalFiles,destDir):
+        cmdUtils = CommandUtils()
+        index=0
+        for source in listAdditionalFiles:
+            if os.path.exists(source):
+                shutil.copytree(source, destDir+str(index))
+                index = index + 1
+
+    def buildRPMSForGivenPackage(self,package,chrootID,listBuildOptionPackages,pkgBuildOptionFile,destLogPath=None):
         self.logger.info("Building rpm's for package:"+package)
 
         listSourcesFiles = constants.specData.getSources(package)
@@ -152,11 +162,31 @@ def buildRPMSForGivenPackage(self,package, chrootID,destLogPath=None):
 #        if os.geteuid()==0:
         self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath)
         self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath)
-
+
+        listAdditionalFiles = []
+        macros = []
+        if package in listBuildOptionPackages:
+            jsonData = open(pkgBuildOptionFile)
+            pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict)
+            jsonData.close()
+            pkgs_sorted = pkg_build_option_json.items()
+            for pkg in pkgs_sorted:
+                p = str(pkg[0].encode('utf-8'))
+                if p == package:
+                    filelist = pkg[1]["files"]
+                    for f in filelist:
+                        listAdditionalFiles.append(str(f.encode('utf-8')))
+                    macrolist = pkg[1]["macros"]
+                    for macro in macrolist:
+                        macros.append(str(macro.encode('utf-8')))
+
+            chrootAdditionalPath=chrootID+constants.topDirPath+"/ADDITIONAL"
+            self.copyAdditionalBuildFiles(listAdditionalFiles,chrootAdditionalPath)
+
         listRPMFiles=[]
         listSRPMFiles=[]
         try:
-            listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath, chrootCmd)
+            listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath,chrootCmd,package,macros)
         except Exception as e:
             self.logger.error("Failed while building rpm:"+package)
             raise e
@@ -176,11 +206,13 @@ def buildRPMSForGivenPackage(self,package, chrootID,destLogPath=None):
             SourcePackageInfo.addSRPMData(package,version,release,arch,srpmName)
 
 
-    def buildRPM(self,specFile,logFile,chrootCmd):
+    def buildRPM(self,specFile,logFile,chrootCmd,package,macros):
 
         rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption
         if not constants.rpmCheck:
             rpmBuildcmd+=" "+self.rpmbuildNocheckOption
+        for macro in macros:
+            rpmBuildcmd+=' --define \\\"%s\\\"' % macro
         rpmBuildcmd+=" "+self.rpmbuildBuildNum+" "+self.rpmbuildReleaseVer
         rpmBuildcmd+=" "+specFile
 

diff --git a/support/package-builder/ThreadPool.py b/support/package-builder/ThreadPool.py
@@ -7,6 +7,8 @@ class ThreadPool(object):
     inactiveWorkerThreads=[]
     mapPackageToCycle={}
     listAvailableCyclicPackages=[]
+    listBuildOptionPackages=[]
+    pkgBuildOptionFile=""
     logger=None
     statusEvent=None
 
@@ -27,7 +29,14 @@ def getAllWorkerObjects():
 
     @staticmethod
     def addWorkerThread(workerThreadName):
-        workerThread = WorkerThread.WorkerThread(ThreadPool.statusEvent,workerThreadName,ThreadPool.mapPackageToCycle,ThreadPool.listAvailableCyclicPackages,ThreadPool.logger)
+        workerThread = WorkerThread.WorkerThread(
+				ThreadPool.statusEvent,
+				workerThreadName,
+				ThreadPool.mapPackageToCycle,
+				ThreadPool.listAvailableCyclicPackages,
+				ThreadPool.logger,
+				ThreadPool.listBuildOptionPackages,
+				ThreadPool.pkgBuildOptionFile)
         ThreadPool.mapWorkerThreads[workerThreadName]=workerThread
 
     @staticmethod

diff --git a/support/package-builder/ToolChainUtils.py b/support/package-builder/ToolChainUtils.py
@@ -94,10 +94,10 @@ def installToolChain(self,chrootID):
                 rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
                 if rpmFile is None:
                     if package == "util-linux-devel":
-                        self.logger.info("No old verion of util-linux-devel exists, skip until the new version is built")
+                        self.logger.info("No old version of util-linux-devel exists, skip until the new version is built")
                         continue
                     if package == "flex-devel":
-                        self.logger.info("No old verion of flex-devel exists, skip until the new version is built")
+                        self.logger.info("No old version of flex-devel exists, skip until the new version is built")
                         continue
 
                     self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
@@ -163,7 +163,7 @@ def findRPMFileInGivenLocation(self,package,rpmdirPath):
             self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package)
             return None
 
-    def buildCoreToolChainPackages(self):
+    def buildCoreToolChainPackages(self, listBuildOptionPackages, pkgBuildOptionFile):
         self.logger.info("Building core tool chain packages.....")
         chrootID=None
         try:
@@ -184,7 +184,7 @@ def buildCoreToolChainPackages(self):
                     raise Exception("creating chroot failed")
                 self.installToolChainRPMS(chrootID)
                 pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
-                pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath)
+                pkgUtils.buildRPMSForGivenPackage(package, chrootID, listBuildOptionPackages, pkgBuildOptionFile, destLogPath)
                 chrUtils.destroyChroot(chrootID)
                 chrootID=None
             self.logger.info("Successfully built toolchain")
@@ -209,10 +209,10 @@ def installToolChainRPMS(self,chrootID):
                 rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
                 if rpmFile is None:
                     if package == "util-linux-devel":
-                        self.logger.info("No old verion of util-linux-devel exists, skip until the new version is built")
+                        self.logger.info("No old version of util-linux-devel exists, skip until the new version is built")
                         continue
                     if package == "flex-devel":
-                        self.logger.info("No old verion of flex-devel exists, skip until the new version is built")
+                        self.logger.info("No old version of flex-devel exists, skip until the new version is built")
                         continue
                     self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
                     raise Exception("Input Error")

diff --git a/support/package-builder/WorkerThread.py b/support/package-builder/WorkerThread.py
@@ -5,13 +5,15 @@
 
 class WorkerThread(threading.Thread):
 
-    def __init__(self,event,name,mapPackageToCycle,listAvailableCyclicPackages,logger):
+    def __init__(self,event,name,mapPackageToCycle,listAvailableCyclicPackages,logger,listBuildOptionPackages,pkgBuildOptionFile):
         threading.Thread.__init__(self)
         self.statusEvent=event
         self.name=name
         self.mapPackageToCycle=mapPackageToCycle
         self.listAvailableCyclicPackages=listAvailableCyclicPackages
         self.logger=logger
+        self.listBuildOptionPackages=listBuildOptionPackages
+        self.pkgBuildOptionFile=pkgBuildOptionFile
 
     def run(self):
         buildThreadFailed=False
@@ -23,7 +25,7 @@ def run(self):
             if pkg is None:
                 break
             self.logger.info("Thread "+self.name+" is building package:"+ pkg)
-            pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,"build-"+pkg)
+            pkgBuilder = PackageBuilder(self.mapPackageToCycle,self.listAvailableCyclicPackages,self.listBuildOptionPackages,self.pkgBuildOptionFile,"build-"+pkg)
             t = threading.Thread(target=pkgBuilder.buildPackageThreadAPI,args=(pkg,outputMap,pkg))
             t.start()
             t.join()
@@ -51,4 +53,4 @@ def run(self):
 
 
 
-
+