diff options
Diffstat (limited to 'utils/source_builder')
| -rw-r--r-- | utils/source_builder/P4.py | 150 | ||||
| -rw-r--r-- | utils/source_builder/RunTestScripts.py | 32 | ||||
| -rw-r--r-- | utils/source_builder/SystemHelpers.py | 30 | ||||
| -rw-r--r-- | utils/source_builder/main_build.cfg | 61 | ||||
| -rw-r--r-- | utils/source_builder/rel_build.cfg | 55 | ||||
| -rw-r--r-- | utils/source_builder/setupVPC.bat | 12 | ||||
| -rw-r--r-- | utils/source_builder/source_build.py | 585 | ||||
| -rw-r--r-- | utils/source_builder/staging_build.cfg | 55 |
8 files changed, 980 insertions, 0 deletions
diff --git a/utils/source_builder/P4.py b/utils/source_builder/P4.py new file mode 100644 index 0000000..0eac203 --- /dev/null +++ b/utils/source_builder/P4.py @@ -0,0 +1,150 @@ +import sys, os, string, re, time, smtplib, getopt + +# syncs the current clientspec +def Sync( szP4SrcFilesToWatch, bForce ): + print "syncing to files " + szP4SrcFilesToWatch + "..." + sForce = "" + if bForce: + sForce = "-f " + aszSyncLines = string.split( szP4SrcFilesToWatch, ";" ) + + for szLine in aszSyncLines: + if szLine: + print "p4 sync " + sForce + szLine + os.popen('p4 sync ' + sForce + szLine) + print " sync completed" + +# current changelist number for this clientspec +def SubmittedChangelist( szP4SrcFilesToWatch ): + szChangeText = os.popen('p4 changes -m1 -s submitted ' + szP4SrcFilesToWatch).read() + line = [] + line = string.split( szChangeText ) + if len(line) > 0: + return line[1] + +# returns a set of changes of range [start, end] +def GetChangelistRange(start, end, szP4SrcFilesToWatch): + if start and end: + szChangeText = os.popen('p4 changes -m10 ' + szP4SrcFilesToWatch + '@' + start + ',' + end).read() + return string.split(szChangeText, '\n'); + szResult = [] + return szResult + +# returns the raw text of a set of the most recent submissions +def GetRecentCheckins( szP4SrcFilesToWatch ): + szChangeText = os.popen('p4 changes -m5 -s submitted ' + szP4SrcFilesToWatch).read() + return "Most recent checkins:\n" + szChangeText + +# perforce counter access +def GetCounter(counter): + return string.split(os.popen('p4 counter ' + counter).read(), '\n')[0] + +#returns the raw text of all unverified checkins +def GetUnverifiedCheckins( szP4SrcFilesToWatch, szVerifiedCounter, szChangeCounter ): + szCounter = GetCounter( szVerifiedCounter ) + szLastVerified = str( int( szCounter ) + 1 ) + szCurrentChange = GetCounter( szChangeCounter ) + szChangeText = os.popen('p4 changes -s submitted ' + szP4SrcFilesToWatch + '@' + szLastVerified + ',@' + szCurrentChange).read() + return "Unverified checkins:\n" + szChangeText + +# extracts an email from a "changes" output line +def GetEmailFromChangeLine(changeline): + change = (string.split(changeline, ' '))[1] + output = os.popen('p4 change -o ' + change).read() + user = string.split(string.split(output, "User:")[2])[0] + output = os.popen('p4 user -o ' + user).read() + return string.split(string.split(output, "Email:")[2])[0] + + +# perforce counter setting +def SetCounter(counter, value): + os.popen('p4 counter ' + counter + ' ' + value).read() + +# checks to see if any update is currently available +def AnyNewCheckins( szChangeCounter, szP4SrcFilesToWatch ): + szChange = GetCounter(szChangeCounter) + if not szChange: + #is this the problem? Every night all these things fail. + return 0 + return szChange <> SubmittedChangelist( szP4SrcFilesToWatch ) + +def LockMutex( szMutex ): + szLogLines = os.popen('newp4mutex lock ' + szMutex + ' 3 steambuilder perforce:1666').read() + count = string.count( szLogLines, '\n' ) + if ( count < 3 ): + print szLogLines + print "HERE IS THE WEIRD LOCK ERROR!" + print szMutex + "mutex lock failed." + return 0 + szT = string.split(szLogLines, '\n')[2] + successLine = string.split( szT, ' ') + if successLine[0] == "Success:": + print szMutex + " mutex locked." + return 1 + else: + print szMutex + " mutex lock failed." + return 0 + +def LockMutexOld( szMutex ): + szLogLines = os.popen('p4mutex lock ' + szMutex + ' 3 steambuilder perforce:1666').read() + count = string.count( szLogLines, '\n' ) + if ( count < 3 ): + print szLogLines + print "HERE IS THE WEIRD LOCK ERROR!" + print szMutex + "mutex lock failed." + return 0 + szT = string.split(szLogLines, '\n')[3] + successLine = string.split( szT, ' ') + if successLine[0] == "Success:": + print szMutex + " mutex locked." + return 1 + else: + print szMutex + " mutex lock failed." + return 0 + +def UnlockMutex( szMutex ): + szLogLines = os.popen('newp4mutex release ' + szMutex + ' 3 steambuilder perforce:1666').read() + count = string.count( szLogLines, '\n' ) + if ( count < 3 ): + print szLogLines + print "HERE IS THE WEIRD RELEASE ERROR!" + print szMutex + "mutex release failed." + return 0 + szT = string.split(szLogLines, '\n')[2] + # print szT + successLine = string.split( szT, ' ') + # print successLine + if successLine[0] == "Success:": + print szMutex + " mutex released." + return 1 + else: + print szMutex + " mutex release failed." + return 0 + +def Integrate( szBranch ): + return os.popen('p4 integ -b ' + szBranch + ' -1 -i' ).read() + +def Revert( szFiles ): + return os.popen('p4 revert ' + szFiles).read() + +def Resolve(): + return os.popen('p4 resolve -am').read() + +def Fstat( szFiles ): + szConflicts = os.popen('p4 fstat -Ru ' + szFiles).read() + return szConflicts + +def Changes( szFile, iNumResults ): + return os.popen('p4 changes -m' + iNumResults + ' ' + szFile).read() + +def Query( szMutex ): + szLines = os.popen( 'p4mutex query ' + szMutex ).read() + count = string.count( szLines, 'HELD' ) + if (count == 0 ): + return 1 + else: + print "Lock held" + return 0 + +def SetClient( szClient ): + os.environ['P4CLIENT'] = szClient diff --git a/utils/source_builder/RunTestScripts.py b/utils/source_builder/RunTestScripts.py new file mode 100644 index 0000000..a7f7b49 --- /dev/null +++ b/utils/source_builder/RunTestScripts.py @@ -0,0 +1,32 @@ + +import sys, os, string, re, time, smtplib, getopt, P4, SystemHelpers + +def RunResultingTestFiles(listTestFiles, szErrorResults ): + while len(listTestFiles): + szFilename = listTestFiles.pop() + os.system( szFilename ) + szErrorResult = os.popen( "type errors.txt" ).read() + if szErrorResult: + szErrorResult = "Script failed: " + szFilename + "\n" + szErrorResult + os.remove("errors.txt") + szErrorResults += szErrorResult + return szErrorResults + + +if __name__ == '__main__': + szErrorResults = "" + SystemHelpers.ChangeDir("\\src\\unittests\\autotestscripts\\") + os.environ['PATH'] += os.pathsep + "d:\\main\\src\\devtools\\bin\\" + \ + os.pathsep + "d:\\main\\game\\bin\\" + listTestFiles = SystemHelpers.ListFiles(".py") + szErrorResults += RunResultingTestFiles(listTestFiles, szErrorResults) + listTestFiles = SystemHelpers.ListFiles(".cmd") + szErrorResults += RunResultingTestFiles(listTestFiles, szErrorResults) + listTestFiles = SystemHelpers.ListFiles(".bat") + szErrorResults += RunResultingTestFiles(listTestFiles, szErrorResults) + listTestFiles = SystemHelpers.ListFiles(".pl") + szErrorResults += RunResultingTestFiles(listTestFiles, szErrorResults) + + + print szErrorResults + diff --git a/utils/source_builder/SystemHelpers.py b/utils/source_builder/SystemHelpers.py new file mode 100644 index 0000000..9a3ecd7 --- /dev/null +++ b/utils/source_builder/SystemHelpers.py @@ -0,0 +1,30 @@ +import os + +szBaseDir = os.getcwd() + +# changes to a new directory, independent of where we are on the drive +def ChangeDir(szDest): + os.chdir(szBaseDir + szDest) + +# deletes a file of disk, ignoring any assert if the file doesn't exist +def DeleteFile(szFile): + try: + os.unlink(szFile) + except OSError, e: + pass + +# returns true if a file exists on disk +def FileExists(szFile): + try: + os.stat(szFile) + return 1 + except OSError, e: + return + +def ListFiles(szExtension): + try: + szAllFiles = os.listdir(os.getcwd()) + return szAllFiles + except OSError, e: + return + diff --git a/utils/source_builder/main_build.cfg b/utils/source_builder/main_build.cfg new file mode 100644 index 0000000..124e50f --- /dev/null +++ b/utils/source_builder/main_build.cfg @@ -0,0 +1,61 @@ +# 1: disables reporting to SRCDEV; only reports to admin +test 0 +# 1: enables testing after build is complete +run_tests 1 +# 1: enables syncing to perforce +sync 1 +# 1: require mutex to be free before proceeding to build +lock_mutex 0 +# 1: build binaries +build 1 +# 1: enables debug output +debug 0 +# 1: script builds immediately upon start, regardless of presence of perforce state +dev 1 +# 1: build shaders +shaders 0 + +# email alias of users interested in build failures +email_alias [email protected] +# address to auto-email if the script fails for some reason, test output +admin_email [email protected] +
# set to hostname of machine running an SMTP server +mail_host exchange2.valvesoftware.com +# email address that the failure mails come from; TODO: Get buildmachine email +sender_email [email protected] + +# executable for compilation; devenv: .Net, BuildConsole: IncrediBuild +build_exe BuildConsole +# build type; /rebuild or /build +build_type /rebuild +# should be empty for devenv, /all for buildconsole +build_flags + +# name of testing file, located in .\Build Machine Tests\ +test_file mainTests.txt +# name of generated log file; currently not used +log_file buildlog.txt +# place to copy log file to reference in failure email +error_dir \\\\jason\\shared\\ +# directory where the testing files are located +test_dir \\Build Machine Tests + +# path to src files to watch +src_files //ValveGames/main/src/... +# paths to files to force sync +force_files //ValveGames/main/src/... +force_files //ValveGames/main/game/bin/... +# paths to files to sync (no force) +sync_files //ValveGames/main/content/tf/materialsrc/... + + +# name of perforce mutex to wait for +mutex main_src +# perforce counter containg the changelist number we're verifying +change_counter main_changelist +# perforce counter the last changelist number we have verified +verify_counter main_verified +# name of build for human-read output +build_name Main +#perforce client used +client_name SourceBuildMachine diff --git a/utils/source_builder/rel_build.cfg b/utils/source_builder/rel_build.cfg new file mode 100644 index 0000000..730eecb --- /dev/null +++ b/utils/source_builder/rel_build.cfg @@ -0,0 +1,55 @@ +# 1: disables reporting to SRCDEV; only reports to admin +test 0 +# 1: enables testing after build is complete +run_tests 0 +# 1: enables syncing to perforce +sync 1 +# 1: require mutex to be free before proceeding to build +lock_mutex 1 +# 1: build binaries +build 1 +# 1: enables debug output +debug 0 +# 1: script builds immediately upon start, regardless of presence of perforce state +dev 1 +# 1: build shaders +shaders 0 + +# email alias of users interested in build failures +email_alias [email protected] +# address to auto-email if the script fails for some reason, test output +admin_email [email protected] +
# set to hostname of machine running an SMTP server +mail_host exchange2.valvesoftware.com +# email address that the failure mails come from; TODO: Get buildmachine email +sender_email [email protected] + +# executable for compilation; devenv: .Net, BuildConsole: IncrediBuild +build_exe BuildConsole +# build type; /rebuild or /build +build_type /rebuild +# should be empty for devenv, /all for buildconsole +build_flags /all + +# name of testing file, located in .\Build Machine Tests\ +test_file relTests.txt +# name of generated log file; currently not used +log_file buildlog.txt +# place to copy log file to reference in failure email +error_dir \\\\jason\\shared\\ +# directory where the testing files are located +test_dir \\Build Machine Tests + +# path to src files to watch +src_files //ValveGames/rel/hl2/src/... +# path to bin files to watch +bin_files //ValveGames/rel/hl2/game/... + +# name of perforce mutex to wait for +mutex rel_src +# perforce counter containg the changelist number we're verifying +change_counter rel_changelist +# perforce counter the last changelist number we have verified +verify_counter rel_verified +# name of build for human-read output +build_name Rel diff --git a/utils/source_builder/setupVPC.bat b/utils/source_builder/setupVPC.bat new file mode 100644 index 0000000..dcbb1c2 --- /dev/null +++ b/utils/source_builder/setupVPC.bat @@ -0,0 +1,12 @@ +copy /y .\game\bin\tier0.dll .\src\devtools\bin +cd .\src\devtools\bin\ +vpc +tools +cd ..\..\..\ +devenv .\src\utils\vpc\vpc.vcproj /rebuild "release|win32" +devenv .\src\utils\vpc\vpc.vcproj /build "release|win32" +copy /y .\game\bin\tier0.dll .\src\devtools\bin +cd .\src\devtools\bin\ +vpc +everything /allgames +vpc +console /x360 /allgames +cd ..\..\..\ + diff --git a/utils/source_builder/source_build.py b/utils/source_builder/source_build.py new file mode 100644 index 0000000..e6ba7e2 --- /dev/null +++ b/utils/source_builder/source_build.py @@ -0,0 +1,585 @@ +#====== Copyright 1996-2005, Valve Corporation, All rights reserved. ======= +# +# Purpose: Syncs, builds and runs tests on the steam code in +# the currently active p4 clientspec +# +#============================================================================= + +# INSTALLATION INSTRUCTIONS: +# +# 1. add the vc compiler solution compiler devenv.exe to the path (defaults to being in "C:\\Program Files\\Microsoft Visual Studio .NET 2003\\Common7\\IDE\\devenv.exe") +# 2. create a P4 clientspec that contains the steam code, eg. //steam/main/... +# 3. make that the default clientspec +# 4. sync to it +# 5. run this build script from inside that directory + + +import sys, os, string, re, time, smtplib, getopt, P4, SystemHelpers + +# config options +g_bTest = 0 # 1: disables reporting to SRCDEV; only reports to admin +g_bRunTests = 0 # 1: enables testing after build is complete +g_bSync = 0 # 1: enables syncing to perforce +g_bLockMutex = 0 # 1: require mutex to be free before proceeding to build +g_bBuild = 0 # 1: build binaries +g_bDebug = 0 # 1: enables debug output +g_bDev = 0 # 1: script builds immediately upon start, regardless of presence of perforce state +g_bShaders = 0 +g_bXBOX = 0 + +g_szEmailAlias = "" # email alias of users interested in build failures +g_szAdministrator = "" # address to auto-email if the script fails for some reason, test output +g_szMailhost = "" # set to hostname of machine running an SMTP server +g_szSenderEmail = "" # email address that the failure mails come from; TODO: Get buildmachine email + +g_szBuildExe = "" # executable for compilation; devenv: .Net, BuildConsole: IncrediBuild +#g_szBuildExe = "devenv" # executable for compilation; devenv: .Net, BuildConsole: IncrediBuild +g_szBuildType = "" # build type; /rebuild or /build +g_szBuildFlags = "" # should be empty for devenv, /all for buildconsole + +g_szTestingFile = "" # name of testing file, located in .\Build Machine Tests\ +g_szLocalLogFile = "" # name of generated log file; currently not used +g_szPublishedErrorsDir = "" # place to copy log file to reference in failure email +g_szTestDirectory = "" # directory where the testing files are located + +g_szBaseDir = os.getcwd() # directory from which the script is launched + +g_szP4SrcFilesToWatch = "" # path to src files to watch +g_szP4ForceFilesToWatch = "" # path to bin files to watch +g_szP4SyncFilesToWatch = "" +g_szP4Mutex = "" # name of perforce mutex to wait for +g_szP4ChangeCounter = "" # perforce counter containg the changelist number we're verifying +g_szP4VerifiedCounter = "" # perforce counter the last changelist number we have verified + +g_nP4MostRecentCheckin = 0 # used to keep track of current checkin +g_nP4LastVerifiedCheckin = 0 # used to keep track of last verified checkin + +g_nSleepTimeBetweenChecks = 10 # number of seconds to wait before checking for a free mutex + +g_szBuildName = "" + + +# mails off a success checkin +def SendSuccessEmail( _szEmail, _szChangeNumber ): + cMailport = smtplib.SMTP(g_szMailhost) + szTestString = "" + if g_bTest: + szTestString = "TEST" + print "sending success email to: " + _szEmail + + szMessage = 'From: ' + g_szBuildName + ' Builder ' + ' <' + g_szSenderEmail + '>\n' +\ + 'To: ' + szTestString + _szEmail + '\n' +\ + 'Subject: [ ' + g_szBuildName + ' build successful ]' +\ + '\n' +\ + '\n' +\ + 'Your checkin:\n\n' +\ + _szChangeNumber +\ + '\n\n' +\ + 'has been successfully built and verified against all tests.\n' + + if ( g_bTest | g_bDev ): + cMailport.sendmail( g_szSenderEmail, g_szAdministrator, szMessage ) + else: + cMailport.sendmail( g_szSenderEmail, _szEmail, szMessage ) + cMailport.quit() + +# print of debugging messages +def dPrint( _szText ): + if g_bDebug: + print _szText + +# mails off the current error string +def SendFailureEmail( _szErr ): + cMailport = smtplib.SMTP( g_szMailhost ) + szTestString = "" + if g_bTest: + szTestString = "TEST" + print "sending failure email to: " + g_szEmailAlias + print "failure reason: " + _szErr + + szMessage = 'From: ' + g_szBuildName + ' Builder' + ' <' + g_szSenderEmail + '>\n' +\ + 'To: ' + szTestString + g_szEmailAlias + '\n' +\ + 'Subject: [ ' + g_szBuildName + ' branch broken ]' +\ + '\n' +\ + '\n' +\ + _szErr +\ + '\n' +\ + P4.GetUnverifiedCheckins( g_szP4SrcFilesToWatch, g_szP4VerifiedCounter, g_szP4ChangeCounter ) + + if g_bTest | g_bDev: + cMailport.sendmail( g_szSenderEmail, g_szAdministrator, szMessage ) + else: + cMailport.sendmail( g_szSenderEmail, szTestString + g_szEmailAlias, szMessage ) + cMailport.quit() + +# use to email the admin about any unexpected errors that occur +def ComplainToAdmin( _szErr ): + cMailport = smtplib.SMTP(g_szMailhost) + print "sending script failure email to: " + g_szAdministrator + print "failure reason: |" + _szErr + "|" + + szMessage = 'From: ' + g_szBuildName + " Builder" + ' <' + g_szSenderEmail + '>\n' +\ + 'To: ' + g_szAdministrator + '\n' +\ + 'Subject: [ ' + g_szBuildName + ' builder build script error ]' +\ + '\n' +\ + '\n' +\ + 'error reason:\n' +\ + _szErr +\ + '\n' + + cMailport.sendmail(g_szSenderEmail, g_szAdministrator, szMessage) + cMailport.quit() + +# parses out the reason for failure from a test +def GetTestFailureReason( _nBuild ): + + # start our error message + szMsg = "Test log file:\n" + + # make a directory to copy the error files into, based on the build config and the p4 changelist number + nBuildNum = P4.GetCounter(g_szP4ChangeCounter) + szErrDir = g_szPublishedErrorsDir + nBuildNum + "\\" + _nBuild + os.popen("mkdir " + szErrDir, "r").read() + + # copy all the files to the errors dir + os.popen("copy " + _nBuild + "\\*.*" + " " + szErrDir + " /Y", "r").read() + # add the error log to the failure message + szMsg += " " + szErrDir + "\\" + g_szLocalLogFile + "\n" + + # add the minidumps to the failure message + rgMinidumps = string.split( os.popen("dir " + _nBuild + "\\*.mdmp /b", "r").read(), "\n" ) + if len(rgMinidumps) > 1: + szMsg += "\nCrash dump:\n" + for szMinidump in rgMinidumps: + if len(szMinidump) > 16: + szMsg += " " + szErrDir + "\\" + szMinidump + "\n" + szMsg += "To view the minidump, click the link above then click \"open\" on the next dialog.\nHit \"F5\" in the now open debugger. When it asks for the source code, change the start of the suggested path from \"c:\\\" to \"\\\\steam3builder\\\".\n" + + # delete the minidumps, so they don't confuse us next time + os.popen("del " + _nBuild + "\\*.mdmp", "r").read() + + + # parse out the error from the logfile + szLog = os.popen( "type " + _nBuild + "\\" + szLocalLogFile, "r").read() + szLogLines = [] + szLogLines = string.split( szLog, "\n" ) + bFoundErr = 0 + for szLine in szLogLines: + aszToken = string.split(szLine, ' ') + if len( aszToken ) > 3: + if aszToken[0] == "***" and aszToken[1] == "TEST" and aszToken[2] == "FAILED": + # probably an error line, add to the list + szMsg += szLine + szMsg += "\n" + bFoundErr = 1 + + # if we haven't found any error lines, use the last line of the file + if bFoundErr == 0 and len(szLogLines) > 0: + szMsg += szLogLines[len(szLogLines) - 1] + + return szMsg + + +# performs a single test run of the specified exe with the specified test parameters +def RunTest( _szCommand ): + + aszParms = string.split( _szCommand, " ", 1) + print "running " + _szCommand + print os.popen( _szCommand, "r" ).read() + + # return success + return aszParms[0] + +def RunCompare( _szCommand ): + szResult = os.popen( _szCommand, "r" ).read() + return szResult + +def SearchFileForErrors( _szFile, _szError ): + print ("Searching " + _szFile + " for occurences of " + _szError + ".\n") + bIncludeNext = 0 + szErrorResults = "" + aszFileLines = string.split( os.popen( "type " + _szFile, "r").read(), "\n" ) + + for szLine in aszFileLines: + if ( bIncludeNext == 1 ): + szErrorResults += szLine + "\n" + bIncludeNext = 0 + aszParms = string.split(szLine, " ", 1) + if (aszParms[0] == _szError): + #there is a UnitTest error, warning, or assert + szErrorResults += "\n" + szLine + "\n" + bIncludeNext = 1; + return szErrorResults + +# runs a set of tests as defined by the test script file +def RunTestScript( ): + print ("Enter Testing") + # make sure we're in the right dir + SystemHelpers.ChangeDir("..") + # load the script + szReturn = os.popen( "RunTestScripts.py" ).read() + return szReturn + +# runs a single build, and runs the tests on the build if specified +def RunBuild( _szBuild ): + # launch devstudio to build the solution + szCmd = _szBuild + aszToken = string.split(szCmd, " ", 3) + + if aszToken[0] == "devenv": + #we are building this. Find out what it is. + if aszToken[2] == "/project": + #building a project. Grab it. + aszToken = string.split(szCmd, " ", 5) + szCmd = g_szBuildExe + " " + aszToken[1] + " /project " + aszToken[3] + " " + g_szBuildType + " " + aszToken[5] + else: + #not build a specific project. Probably everything under lostcoast. + szCmd = g_szBuildExe + " " + aszToken[1] + " " + g_szBuildType + " " + aszToken[3] + " " + g_szBuildFlags + else: + #didn't see the devenv line; not building this but it isn't an error either. + return "" + + print "building: " + szCmd + szOutput = os.popen(szCmd, "r").read() + # parse the output for any errors + aszOutputLines = string.split(szOutput, "\n") + bSuccess = 1 + szBuildErr = "\n\n\nError building configuration " + _szBuild + ":\n" + + for szLine in aszOutputLines: + if g_bDebug: + print szLine; + aszTokens = string.split(string.lstrip(szLine), ' ', 4) + if len(aszTokens) > 1: + if aszTokens[0] == "--------------------Configuration:": + bPrintedProject = 0 + szProject = aszTokens[1] + if aszTokens[1] == ":" and aszTokens[2] == "error" and aszTokens[3] != "PRJ0019": + # probably an error line, add to the list + count = string.count( szBuildErr, aszTokens[4]) + if ( count == 0 ): + if ( bPrintedProject == 0 ): + szBuildErr += "Project: " + szProject + "\n" + bPrintedProject = 1 + szBuildErr += szLine + "\n" + #err2 += szLine + "\n" + bSuccess = 0 + if aszTokens[1] == ":" and aszTokens[2] == "error" and aszTokens[3] == "PRJ0019": + # the delete error line, add to the list + szBuildErr += "IGNORE: " + szBuildErr += szLine + "\n" + #err2 += szLine + "\n" + aszTokens = string.split(string.lstrip(szLine), ' ') + if len(aszTokens) > 4: + # can't do this: the weird delete error will trigger this and we need to ignore that + # check that the "Rebuild All: x succeeded, y failed, z skipped line says no failures + #if szT[0] == "Rebuild" and szT[3] rrorMessages + "The " + szTestName + " test failed\nThe error is " + szCompareLine + #if szErrorMessages== "succeeded," and szT[5] == "failed," and not szT[4] == "0": + # failure + # bSuccess = 0 + # check for linker errors + if aszTokens[0] == "LINK" and aszTokens[1] == ":" and aszTokens[2] == "fatal" and aszTokens[3] == "error": + if ( bPrintedProject == 0 ): + szBuildErr += "Project: " + szProject + "\n" + bPrintedProject = 1 + # linker error line, add to the list + szBuildErr += szLine + "\n" + #err2 += szLine + "\n" + bSuccess = 0 + # can't do this either: Delete file problem + # check the standard error dealie. + # if szT[1] == '-' and szT[3] == "error(s)," and szT[2] != '0': + #we have a build error here + # err += szLine + # err += "\n" + #check for fatal error + if aszTokens[2] == "fatal" and aszTokens[3] == "error": + if ( bPrintedProject == 0 ): + szBuildErr += "Project: " + szProject + "\n" + bPrintedProject = 1 + #fatal error + szBuildErr += szLine + "\n" + #err2 += szLine + "\n" + bSuccess = 0 + + # return immediately if failed + if not bSuccess: + print szBuildErr + szBuildErr + "\n\n" + #ComplainToAdmin(err2) + return szBuildErr + + return "" + +def RunBuildBatch(): + bSuccess = 1 + #aszBatchBuildLines = string.split( os.popen( "type " + szBatchFile, "r").read(), "\n" ) + #bIsDevLine = 0 + szBuildErrs = "" + szBuildResult = RunBuild( "devenv everything.sln /build \"debug|win32\" " ) + + szBuildResult += RunBuild( "devenv everything.sln /build \"release|win32\" " ) + szTestResult = RunTestScript() + if szBuildResult != "": + szBuildErrs += szBuildResult + bSuccess = 0 + if szTestResult != "\n": + szBuildErrs += szTestResult + bSuccess = 0 + if not bSuccess: + SendFailureEmail(szBuildErrs) + return bSuccess + + +# runs all the builds +def RunAllBuilds(): + bSuccess = 1 + szBuildErrs = "" + SystemHelpers.ChangeDir("\game") + if g_bBuild: + SystemHelpers.ChangeDir("\src") + + # build the shadercompiler and all shaders + if g_bShaders: + print( "Compiling shadercompile" ) + os.system( "devenv shadercompile.sln /rebuild release > silence" ) + # should check here to make sure the shadercompile worked + SystemHelpers.ChangeDir("\\src\\materialsystem\\stdshaders") + print( "Building shaders" ) + child_stdin, child_stdout, child_stderr = os.popen3( "buildallshaders.bat" ) + print( child_stdout.read() ) + szSomeShaderErr = child_stderr.read() + szShaderErrors = "" + aszShaderLines = string.split( szSomeShaderErr, "\n" ) + for szLine in aszShaderLines: + dPrint( szLine ) + nCount = string.count( szLine, 'U1073:' ) + if nCount > 0: + aszToken = string.split( szLine, " " ) + szShaderName = aszToken[9] + szShaderErrors = szShaderErrors + "The shader file " + szShaderName + " is missing and failed during buildallshaders.bat.\n" + if szShaderErrors: + SendFailureEmail( szShaderErrors ) + + SystemHelpers.ChangeDir("\\src") + bSuccess = bSuccess & RunBuildBatch(); + +#XBOX Section + if g_bXBOX: + szXBoxOutput = RunBuild( "devenv source_x360.sln /rebuild \"release|xbox 360\"" ) + if "\n\n\nError building configuration " + "devenv source_x360.sln /rebuild release" + ":\n" != szXBoxOutput: + bSuccess = 0 + SendFailureEmail( szXBoxOutput ) + #delete tier0.dll + + + if g_bRunTests: + szTestErrors = RunTestScript() + if szTestErrors != "\n": + # success = 0 + ComplainToAdmin( szTestErrors ) + return bSuccess + +# builds from a local branch and runs a subset of tests +def PerformSourceBuild(): + + # build and test in each configuration + if not RunAllBuilds(): + print "Source build failed" + return 0 + + print "Source build: SUCCESS" + return 1 + + +# syncs, builds, runs tests +def PerformDailyBuild(): + print " changes detected, starting daily build" + # update the counter to be what we're verifying + change = P4.SubmittedChangelist( g_szP4SrcFilesToWatch ) + g_nP4MostRecentCheckin = change + g_nP4LastVerifiedCheckin = P4.GetCounter(g_szP4VerifiedCounter) + if g_nP4MostRecentCheckin and g_nP4LastVerifiedCheckin: + print "Most recent checkin is " + g_nP4MostRecentCheckin + "\n" + print "Last verified checkin is " + g_nP4LastVerifiedCheckin + "\n" + # the p4 command can occasionally fail to deliver a valid changelist number, unclear why + # can't update the counter, it just means we'll run twice + if change: + P4.SetCounter(g_szP4ChangeCounter, change) + # sync to the new files + if ( g_bSync ): + SystemHelpers.ChangeDir("\\src") + print( "Cleaning\n" ) + os.system("cleanalltargets.bat > silence") + SystemHelpers.ChangeDir("\\") + print "Synching force files." + P4.Sync( g_szP4ForceFilesToWatch, 1 ) + print "Synching other files." + P4.Sync( g_szP4SyncFilesToWatch, 0 ) + print( "Setting up VPC" ) + os.system("setupVPC.bat") + + #P4.UnlockMutex(g_szP4Mutex) + # build and test in each configuration + if not RunAllBuilds(): + print "Daily build failed" + return + + # send a success email, from past the last successful checkin to the current + if change: + szVerifiedOrig = P4.GetCounter(g_szP4VerifiedCounter) + if szVerifiedOrig: + szVerifiedPlusOne = str( int( szVerifiedOrig ) + 1 ) + changes = P4.GetChangelistRange(szVerifiedPlusOne, change, g_szP4SrcFilesToWatch ); + for ch in changes: + if len(ch) > 1: + szEmail = P4.GetEmailFromChangeLine(ch) + SendSuccessEmail(szEmail, ch) + #SendSuccessEmail("jason", ch) + # remember this change that we've verified + P4.SetCounter(g_szP4VerifiedCounter, change) + + print "Daily build: AN UNEQUIVOCAL SUCCESS" + +def PrintConfig(): + print("Configuration:") + print("test = " + str(g_bTest)) + print("run_tests = " + str(g_bRunTests)) + print("lock_mutex = " + str(g_bLockMutex)) + print("build = " + str(g_bBuild)) + print("debug = " + str(g_bDebug)) + print("dev = " + str(g_bDev)) + print("shaders = " + str(g_bShaders)) + print("sync = " + str(g_bSync)) + print("email_alias = " + g_szEmailAlias) + print("admin_email = " + g_szAdministrator) + print("mail_host = " + g_szMailhost) + print("sender_email = " + g_szSenderEmail) + print("build_exe = " + g_szBuildExe) + print("build_type = " + g_szBuildType) + print("build_flags = " + g_szBuildFlags) + print("test_file = " + g_szTestingFile) + print("log_file = " + g_szLocalLogFile) + print("error_dir = " + g_szPublishedErrorsDir) + print("test_dir = " + g_szTestDirectory) + print("src_files = " + g_szP4SrcFilesToWatch) + print("force_files = " + g_szP4ForceFilesToWatch) + print("sync_files = " + g_szP4SyncFilesToWatch) + print("mutex = " + g_szP4Mutex) + print("change_counter = " + g_szP4ChangeCounter) + print("verify_counter = " + g_szP4VerifiedCounter) + print("build_name = " + g_szBuildName) + +def ParseConfigFile(configFileName): + aszBatchBuildLines = string.split( os.popen( "type " + configFileName, "r").read(), "\n" ) + global g_bTest + global g_bRunTests + global g_bLockMutex + global g_bBuild + global g_bDebug + global g_bDev + global g_bShaders + global g_bSync + global g_szEmailAlias + global g_szAdministrator + global g_szMailhost + global g_szSenderEmail + global g_szBuildExe + global g_szBuildType + global g_szBuildFlags + global g_szTestingFile + global g_szLocalLogFile + global g_szPublishedErrorsDir + global g_szTestDirectory + global g_szP4SrcFilesToWatch + global g_szP4ForceFilesToWatch + global g_szP4SyncFilesToWatch + global g_szP4Mutex + global g_szP4ChangeCounter + global g_szP4VerifiedCounter + global g_szBuildName + for szLine in aszBatchBuildLines: + aszTokens = string.split(string.lstrip(szLine), ' ', 2) + firstToken = aszTokens[0] + if firstToken == '#' or firstToken == '': + continue + secondToken = aszTokens[1] + if firstToken == "test": + g_bTest = int(secondToken) + elif firstToken == "run_tests": + g_bRunTests = int(secondToken) + elif firstToken == "lock_mutex": + g_bLockMutex = int(secondToken) + elif firstToken == "build": + g_bBuild = int(secondToken) + elif firstToken == "debug": + g_bDebug = int(secondToken) + elif firstToken == "dev": + g_bDev = int(secondToken) + elif firstToken == "shaders": + g_bShaders = int(secondToken) + elif firstToken == "sync": + g_bSync = int(secondToken) + elif firstToken == "email_alias": + g_szEmailAlias = secondToken + elif firstToken == "admin_email": + g_szAdministrator = secondToken + elif firstToken == "mail_host": + g_szMailhost = secondToken + elif firstToken == "sender_email": + g_szSenderEmail = secondToken + elif firstToken == "build_exe": + g_szBuildExe = secondToken + elif firstToken == "build_type": + g_szBuildType = secondToken + elif firstToken == "build_flags": + g_szBuildFlags = secondToken + elif firstToken == "test_file": + g_szTestingFile = secondToken + elif firstToken == "log_file": + g_szLocalLogFile = secondToken + elif firstToken == "error_dir": + g_szPublishedErrorsDir = secondToken + elif firstToken == "test_dir": + g_szTestDirectory = secondToken + elif firstToken == "src_files": + g_szP4SrcFilesToWatch += secondToken + elif firstToken == "force_files": + g_szP4ForceFilesToWatch += secondToken + ";" + elif firstToken == "sync_files": + g_szP4SyncFilesToWatch += secondToken + ";" + elif firstToken == "mutex": + g_szP4Mutex = secondToken + elif firstToken == "change_counter": + g_szP4ChangeCounter = secondToken + elif firstToken == "verify_counter": + g_szP4VerifiedCounter = secondToken + elif firstToken == "build_name": + g_szBuildName = secondToken + PrintConfig() + +#----------------------------------------------------------------------------- +# Main +#----------------------------------------------------------------------------- +if __name__ == '__main__': + try: + print "----------------------------------------------------" + print g_szBuildName + " BUILD SCRIPT STARTED" + ParseConfigFile(sys.argv[1]) + + while 1: + if (g_bDev | P4.AnyNewCheckins( g_szP4ChangeCounter, g_szP4SrcFilesToWatch )): + print "Changes Detected.\n" + if ( (g_bTest & ~g_bLockMutex) | ~g_bLockMutex | P4.Query(g_szP4Mutex) ): + PerformDailyBuild() + g_bDev = 0 + print "" + print "------------------------------------------" + print "waiting for changes to be detected..." + else: + time.sleep( g_nSleepTimeBetweenChecks - ( g_bTest * g_nSleepTimeBetweenChecks )) + else: + time.sleep( g_nSleepTimeBetweenChecks ) + except RuntimeError, e: + ComplainToAdmin(e) + diff --git a/utils/source_builder/staging_build.cfg b/utils/source_builder/staging_build.cfg new file mode 100644 index 0000000..1a513ea --- /dev/null +++ b/utils/source_builder/staging_build.cfg @@ -0,0 +1,55 @@ +# 1: disables reporting to SRCDEV; only reports to admin +test 0 +# 1: enables testing after build is complete +run_tests 0 +# 1: enables syncing to perforce +sync 1 +# 1: require mutex to be free before proceeding to build +lock_mutex 1 +# 1: build binaries +build 1 +# 1: enables debug output +debug 0 +# 1: script builds immediately upon start, regardless of presence of perforce state +dev 1 +# 1: build shaders +shaders 0 + +# email alias of users interested in build failures +email_alias [email protected] +# address to auto-email if the script fails for some reason, test output +admin_email [email protected] +
# set to hostname of machine running an SMTP server +mail_host exchange2.valvesoftware.com +# email address that the failure mails come from; TODO: Get buildmachine email +sender_email [email protected] + +# executable for compilation; devenv: .Net, BuildConsole: IncrediBuild +build_exe BuildConsole +# build type; /rebuild or /build +build_type /rebuild +# should be empty for devenv, /all for buildconsole +build_flags /all + +# name of testing file, located in .\Build Machine Tests\ +test_file mainTests.txt +# name of generated log file; currently not used +log_file buildlog.txt +# place to copy log file to reference in failure email +error_dir \\\\jason\\shared\\ +# directory where the testing files are located +test_dir \\Build Machine Tests + +# path to src files to watch +src_files //ValveGames/staging/src/... +# path to bin files to watch +bin_files //ValveGames/staging/game/... + +# name of perforce mutex to wait for +mutex staging_src +# perforce counter containg the changelist number we're verifying +change_counter staging_changelist +# perforce counter the last changelist number we have verified +verify_counter staging_verified +# name of build for human-read output +build_name Staging |