[python]代码库
import random
import time
import EnvSetup.EnvSetup
import Library.UFS.TestInstance as TestInstance
# import config.UFS
import os
import copy
import math
import csv
from Library.UFS.ufsDefines import *
from Library.Generic import timer
# Extra Imports
from collections import namedtuple
import collections
class Latency_00_Command_Base(TestInstance.TestInstance):
SIZE_1KB = 1024
SIZE_1MB = 1024 * SIZE_1KB
SIZE_1GB = 1024 * SIZE_1MB
LBA_SIZE = 4 * SIZE_1KB
lbaStepSize = SIZE_1GB / LBA_SIZE
DEBUG = False
vcc = -1
vccq = -1
vccq2 = -1
atariCategory = 'Latency'
atariGroup = 'TSM'
outputUnit = 'ms'
pwrMode = 'Active'
exitOnFailure = False
verbose = False
type = 'Scsi'
def AddCommandArgs(self):
super(Latency_00_Command_Base, self).AddCommandArgs()
# configure divice
self.parser.add_argument('--mediaLunCount', type=int, default=1, help="""configure device with defalut(0ne) lun""")
self.parser.add_argument('--bootLunCount', type=int, default=0, help="""configure device with default(default 0) boot Lun""")
self.parser.add_argument('--competitor', type=bool, default=False, help="""whether the device comes from competitor. defalut False""")
self.parser.add_argument('--tgtProvisionType', type=int, default=3, help="""provisioning type. discard,2,TPRZ=0/erase,3,TPRZ=1""")
# argument for precondition
self.parser.add_argument('--seqTransferTL', type=int, default=0x80, help="""sequential write TL, default 512K""")
self.parser.add_argument('--seqData', type=int, default=0xAA, help="""sequential Write Data""")
self.parser.add_argument('--randNumber', type=int, default=100000, help="""random write cmd count per GB""")
self.parser.add_argument('--randData', type=int, default=0x55, help="""random write data""")
# argument for data collection and flow contorl
self.parser.add_argument('--basicCmdNum', type=int, default=100000, help="""command count for each iteration/GB""")
self.parser.add_argument('--totalCmdNum', type=int, default=100000, help="""Sample points needed for data collection""")
# argument for debug function
# self.parser.add_argument('--specialSaveRawData', type=bool, default=False, help="""For special format save raw data collection""")
self.parser.add_argument('--vuLatencyEn', type=bool, default=False, help="""If using latency VU, should use latency FW. print data profiling debug.""")
self.parser.add_argument('--dumpCache', type=bool, default=False, help="""If necessary, open this. each cache count, script will print command info and vu data""")
self.parser.add_argument('--vuCMDTO', type=float, default=30.0, help="""For read/sync/other command time out parameter. if command > 30ms, timeout""")
self.parser.add_argument('--vuWriteTO', type=float, default=30.0, help="""Just for write, if command > 30ms, timeout""")
self.parser.add_argument('--cacheCount', type=int, default=8, help="""save how many command into cache.""")
# Others
self.parser.add_argument('--defrageTable', type=bool, default=False, help="""dirty card or not""")
def _reconfigDevice(self, bSecureRemovalTypeSet=0, bProvisionTypeReset=3):
if not self.competitor:
self.vuLib.vuCmdtranslate.restoreWriteOnce()
response = self.lun[0].Query.ReadDescriptorSync(0x1, 0, 0, 0x90)
reconfigBuffer = response.DataSegmentBuffer
if reconfigBuffer.GetByte(2) != 0:
reconfigBuffer.SetByte(2, 0)
reconfigBuffer.SetByte(7, bSecureRemovalTypeSet)
self.lun[0].Query.WriteDescriptorSync(0x1, 0, 0, 0x90, reconfigBuffer)
maxAllocUinit = self.lib.query.getdEnhanced1MaxNAllocU()
HighByte = (int(maxAllocUinit) & 0xFF00) >> 8
LowByte = int(maxAllocUinit) & 0xFF
response = self.lun[0].Query.ReadDescriptorSync(1, 0, 0, 0x90)
ReconfigBuf = response.DataSegmentBuffer
for i in range(1, 9):
ReconfigBuf.SetByte(i * 16 + 0xA, bProvisionTypeReset)
ReconfigBuf.SetByte(i * 16, 0)
ReconfigBuf.SetByte(i * 16 + 1, 0)
ReconfigBuf.SetByte(i * 16 + 6, 0)
ReconfigBuf.SetByte(i * 16 + 7, 0)
ReconfigBuf.SetByte(0x16, HighByte)
ReconfigBuf.SetByte(0x17, LowByte)
ReconfigBuf.SetByte(0x10, 1)
ReconfigBuf.SetByte(0x2, 0)
self.lun[0].Query.WriteDescriptorSync(1, 0, 0, 0x90, ReconfigBuf)
def _doBKOPs(self, defautTime=400):
buf = self.mist.Buffer(0x1, LBA_SIZE)
t = timer(defautTime)
while not t.expired:
if self.lib.query.getbBackgroundOpStatus() > 0:
wExceptionEventStatus = self.lib.query.getwExceptionEventStatus()
self.logger.info("send one read command")
self.lun[self.lunID].Read10SimpleSync(0x1, 0x1, buf)
self.logger.info("--------Get wExceptionEventStatus:{}".format(wExceptionEventStatus & 0x4))
time.sleep(60)
else:
break
currentBKOPs = self.lib.query.getbBackgroundOpStatus()
if currentBKOPs == 0 or currentBKOPs == 1:
self.logger.info("current BKOPs has been completed")
else:
self.testFailed('bBackgroundOpStatus is still {} after {}'.format(currentBKOPs, defautTime))
def _newTargetConfg(self, bootOrNot=False):
if not self.competitor:
self.logger.info("########## Create new target configuration descriptor ##########")
if bootOrNot:
bootCapacity = 16
self.lib.lunConfig.createTargetConfig(mediaLunCount=self.mediaLunCount, bootLunCount=self.bootLunCount, setBigLun=True,
useAllCapacity=True, setHighPriorityLun=False,
bProvisioningType=self.tgtProvisionType, bootCapacity_MB=bootCapacity)
else:
self.lib.lunConfig.createTargetConfig(mediaLunCount=self.mediaLunCount, bootLunCount=self.bootLunCount, setBigLun=True,
useAllCapacity=True, setHighPriorityLun=False, bProvisioningType=self.tgtProvisionType)
self.lib.lunConfig.printDescriptor() # Print the new created target descriptor
disabledLun = self.lib.lunConfig.getDisableLunInTargetConfig()
self.logger.info("Disabled LUs are: %s" % disabledLun)
targetCfgOutline = self.lib.lunConfig.getTargetConfigOutline()
for key, value in targetCfgOutline.items():
self.logger.info("Category {} : {}".format(key, value))
# Write the new target configuration descriptor to device
self.lib.lunConfig.writeTargetConfigToDevice()
else:
self.logger.info("the unit comes from competitor")
def _latencyOfPurge(self, purgeTimeOut=None):
"""
Original purgeTimeoOut=4000
Purge all unmaped LBA(s)
NOTE: This latency is in increments of 1ms
:param purgeTimeOut: (int) in seconds the timeout period to wait for the purge command
:return usPurgeTime: (float) command execution time in microseconds (us)
"""
self.logger.info("***********************do purge*********************")
if purgeTimeOut is None:
purgeTimeOut = self.cfg.timeout.purgeTimeout
# Just for leo, to pass this case.
if self.product == "Leo":
purgeTimeOut = 10000
setPurgeResp = self.lib.query.setfPurgeEnable()
if setPurgeResp == 1:
purgeStartTime = time.time()
else:
self.testFailed(
"Set fPurgeEnable flag enable fail, and the fPurgeEnable flag value is {}".format(setPurgeResp))
bPurgeStatus = -1
while bPurgeStatus not in (PURGE_STATUS_STOPPED_BY_HOST, PURGE_STATUS_COMPLETED):
bPurgeStatus = self.lib.query.getbPurgeStatus()
if time.time() - purgeStartTime > purgeTimeOut:
self.testFailed("FAIL bPurgeStatus ({}) not set to {} before timeOut {}".format(setPurgeResp, (
PURGE_STATUS_STOPPED_BY_HOST, PURGE_STATUS_COMPLETED), purgeTimeOut))
purgeEndTime = time.time()
secondsPurgeTime = purgeEndTime - purgeStartTime
usPurgeTime = self.lib.generic.changeUnits("us", "seconds", secondsPurgeTime)
return usPurgeTime
def _eraseFullCardPost(self):
self.logger.info("pre/post condition: format unit and purge")
lunList = self.lib.reportLuns.getAvailableNormalLuns()
for lunID in lunList:
self.lun[lunID].FormatUnitSimpleSync()
self._latencyOfPurge()
def _defragTableVB(self):
tl = 1
buf = self.mist.Buffer(1, 8)
maxLba = self.lun[self.lunID].ReadCapacity10Sync(buf).MaxLba
writeBuffer = self.mist.Buffer(tl, LBA_SIZE)
writeBuffer.Fill(0x55)
for i in range(12):
startLba = random.randint(0, 1020)
while startLba < maxLba:
if startLba + tl > maxLba:
startLba = maxLba - tl
self.lun[self.lunID].Write10Sync(lba=startLba, count=tl, disablePageOut=False, fua=False, fuaNv=False, buf=writeBuffer)
startLba = startLba + 1024
def _sendWriteCMD(self, startLba, endLba, transferLen=1, writeData=0xAA, cmdNumber=0, seq=False, QD=32):
outstandingCmd = {}
responseList = []
timeout = 60
buffers = [self.mist.Buffer(transferLen, LBA_SIZE) for _ in range(QD)]
buffTrk = range(QD)
temp = 0
if seq:
self.logger.debug("sequential write from startLba=--{}-- to endLba=--{}--".format(startLba, endLba))
while (startLba <= endLba):
if startLba + transferLen > endLba:
tempTL = endLba - startLba + 1
else:
tempTL = transferLen
if len(outstandingCmd) < QD:
buffNum = buffTrk.pop()
buf = buffers[buffNum]
buf.Fill(writeData)
usr = self.lun[self.lunID].Write10Simple(startLba, tempTL, buf)
outstandingCmd[usr.SequenceId] = (buffNum)
if len(outstandingCmd) == QD:
startTime = time.time()
while len(responseList) == 0:
elapsedTime = time.time() - startTime
if elapsedTime > timeout:
break
try:
responseList = self.lun[self.lunID].CompletedResponses
except self.mist.ufs.UpiuCommandError as e:
self.testFailed("UpiuCommandError happened {}".format(e))
except Exception as e:
self.testFailed("Exception happened {}".format(e))
for resp in responseList:
sequenceId = resp.SequenceId
buffNum = outstandingCmd.pop(sequenceId)
buffTrk.append(buffNum)
responseList = []
startLba += tempTL
if startLba == endLba + 1:
if 0 < len(outstandingCmd) < QD:
for i in range(len(outstandingCmd)):
self.lun[self.lunID].WaitForOneResponse()
else:
if cmdNumber:
self.logger.debug("random send cmdNumber={} write command in startLba={},endLba={}".format(cmdNumber, startLba, endLba))
while (temp < cmdNumber):
if len(outstandingCmd) < QD:
buffNum = buffTrk.pop()
buf = buffers[buffNum]
buf.Fill(writeData)
tempLba = random.randint(startLba, endLba - transferLen + 1)
usr = self.lun[self.lunID].Write10Simple(tempLba, transferLen, buf)
temp = temp + 1
outstandingCmd[usr.SequenceId] = (buffNum)
if len(outstandingCmd) == QD:
startTime = time.time()
while len(responseList) == 0:
elapsedTime = time.time() - startTime
if (elapsedTime > timeout) and len(responseList) == 0:
self.testFailed("Timeout occurred after %s,no responses returned " % elapsedTime)
break
try:
responseList = self.lun[self.lunID].CompletedResponses
except self.mist.ufs.UpiuCommandError as e:
self.testFailed("UpiuCommandError happened {}".format(e))
except Exception as e:
self.testFailed("Exception happened {}".format(e))
for resp in responseList:
sequenceId = resp.SequenceId
buffNum = outstandingCmd.pop(sequenceId)
buffTrk.append(buffNum)
responseList = []
if temp == cmdNumber:
for i in range(len(outstandingCmd)):
self.lun[self.lunID].WaitForOneResponse()
else:
writeBuffer = self.mist.Buffer(transferLen, LBA_SIZE)
writeBuffer.Fill(writeData)
self.logger.debug("random send write command on current lbaRange until all LBA has been written")
self.lib.writeReadAsync.writeRandom(self.lunID, "Write10", writeBuffer, startLba, endLba, transferLen, QD, fua=True)
def _overwriteCli(self):
self.responseLoggerIsEnabled = False
self.responseLoggerReportUpiu = False
self.responseLoggerReportFailOnly = True
self.needstandardTestStart = False
def _preConditionForSAS(self, SSPreCondition=False):
if self.DEBUG:
self.logger.info('DEBUG running, precondition skipped')
return
self.logger.info("sequential write full card with ChunkSize *** 512KB ***, data is 0xAA")
buff = self.mist.Buffer(1, 8)
endLba = self.lun[self.lunID].ReadCapacity10Sync(buff).MaxLba
self._sendWriteCMD(0, endLba, transferLen=self.seqTransferTL, seq=True)
if not SSPreCondition:
self.logger.info("============Random write 100000 per GB with pattern 0x55============")
for startLba, endLba in self.LbaRange:
self._sendWriteCMD(startLba, endLba, writeData=self.randData, cmdNumber=self.randNumber)
else:
self.logger.info("precondition for SS's test method")
cmdNumber = self.maxLba / self.seqTransferTL
self._sendWriteCMD(0, self.maxLba, transferLen=self.seqTransferTL, writeData=self.randData, cmdNumber=cmdNumber)
if self.defrageTable:
self._defragTableVB()
def _preCondition(self, lunID, loopSize=1000):
"""
Precondition options you can overwrite by children classes.
:param lunID: current lun
:return oldValues: (dict) the values before this method changed them
:return avgTeUnRe: (float) average test unit ready time
:return secIdleBoundary: (float) time to put device in idle mode in seconds
"""
newValues = {}
if hasattr(self, 'newWce') and self.newWce >= 0:
newValues["newWce"] = self.newWce
if hasattr(self, 'newRcd') and self.newRcd >= 0:
newValues["newRcd"] = self.newRcd
if hasattr(self, 'gearHS') and self.gearHS >= 0:
newValues["gearHS"] = self.gearHS
if hasattr(self, 'rateHS') and self.rateHS != "None":
newValues["rateHS"] = self.rateHS
if hasattr(self, 'ufsPwrMode') and self.powerMode >= 0:
newValues["ufsPwrMode"] = self.ufsPwrMode
if hasattr(self, 'laneNum') and self.laneNum >= 0:
newValues["laneNum"] = self.laneNum
if hasattr(self, 'bDataReliability') and self.bDataReliability >= 0:
newValues["bDataReliability"] = self.bDataReliability
newValues["newVoltages"] = self.lib.performanceAdHoc.getVoltages(
vcc=self.vcc, vccq=self.vccq, vccq2=self.vccq2)
oldValues = self.lib.performanceAdHoc.latencyPreCondition(lunID=lunID, newValues=dict(newValues), verbose=False,
exitOnFailure=self.exitOnFailure)
self.logger.parameter(paramName="newValues", value=newValues, category=self.atariCategory, group=self.atariGroup)
self.logger.parameter(paramName="oldValues", value=oldValues, category=self.atariCategory, group=self.atariGroup)
(usAvgTeUnRe, _, _) = self.lib.performanceAdHoc.testUnitReadyLoopAndPrint(lunID=lunID, loopSize=loopSize, verbose=False)
avgTeUnRe = self.lib.generic.changeUnits(self.outputUnit, "us", usAvgTeUnRe)
self.logger.parameter(paramName="Test Unit Ready Average", value=avgTeUnRe, unit=self.outputUnit, category=self.atariCategory, group=self.atariGroup)
if self.pwrMode == "Idle":
(_, rawBoundary, rawUnit) = self.cfg.latency.getLatencyExpected("Typical", "Active to Idle")
idleBoundary = self.lib.generic.changeUnits(self.outputUnit, rawUnit, rawBoundary)
else:
idleBoundary = None
preCondition = namedtuple("preCondition", "oldValues avgTeUnRe secIdleBoundary")
return preCondition(oldValues=oldValues, avgTeUnRe=avgTeUnRe, secIdleBoundary=idleBoundary)
def _dumpCacheData(self):
self.logger.info("===================Dump CMD/VU Buffer=======================")
self.logger.info("NOTE: CMD Info Format: (cmdType, lun, lba, transferLen, taskTag, sequenceId, exec(ms)")
self.logger.info("Debug. cache cmd count:{}".format(len(self.vuCacheDict)))
for cmdKey, vuValue in self.vuCacheDict.items():
tempList = []
tempList.append(cmdKey)
self.logger.info("cmdKey(ms):{}".format(cmdKey))
self.logger.info("vuValue:{}".format(vuValue))
for index, tempValue in enumerate(vuValue):
tempList.append(tempValue)
self._saveRawData(["TimeOut_VU_Get_Latency"], [tempList])
# if the whole cache data are dumped, clear cache.
self.vuCacheDict.clear()
def _vuLatency(self, cmdInfo, checkTO=True):
# cmdInfo = (cmdType, lun, lba, transferLen, taskTag, sequenceId, execTime)
cmdType = cmdInfo[0]
execTime = cmdInfo[-1]
if self.cacheCount:
if len(self.vuCacheDict) >= self.cacheCount:
if self.dumpCache:
self._dumpCacheData()
else:
# if no dump data, then pop one item for the next cmd info.
self.vuCacheDict.popitem(0)
vuData = self.vuLib.vuCmdtranslate.getProfileDataLatency()
self.vuCacheDict[cmdInfo] = vuData
# Check command whether timeout.if yes, test failed immediately.
if cmdType == 'Mode Sense':
self.vuLatencyTO = 1
if cmdType == "Mode Select":
self.vuLatencyTO = 100
self.logger.info("111----set command timeout {}ms for cmdType {}".format(self.vuLatencyTO, cmdType))
if checkTO and execTime >= self.vuLatencyTO:
if self.cacheCount:
self._dumpCacheData()
try:
vuData = self.vuLib.vuCmdtranslate.getProfileDataLatency()
self.logger.info("Oops! Occurred cmd time out!, TargetTO:{} ms".format(self.vuLatencyTO))
self.logger.info("NOTE: CMD Info Format: (cmdType, lun, lba, transferLen, taskTag, sequenceId, exec(ms)")
self.logger.info("CMD Info(ms):{}, ".format(cmdInfo))
self.testFailed("GetProfileData:{}".format(vuData))
except self.mist.ufs.UpiuCommandError as e:
self.logger.info("Get profileData error:{}".format(e))
def _postCondition(self, lunID, oldValues):
"""
Postcondition options you can overwrite by children classes.
:param lunID: current lun
:param oldValues: (dict) the values to set the device back to before ending the next test
"""
self.lib.performanceAdHoc.latencyPostCondition(lunID=lunID, oldValues=oldValues)
def _afterLoopBeforeAveraging(self, usLatencies):
"""
Some scripts need to modify the latencies before proceeding with the test
:param lunID: (int) current lun
:param usLatencies: [int] The latencies from the main function in micro seconds
:returns latencies: [int] The modified latencies
"""
latencies = self.lib.generic.changeUnits(self.outputUnit, "us", usLatencies)
return latencies
def _splitdrive(self, p):
"""Split a pathname into drive and path specifiers. Returns a 2-tuple
"(drive,path)"; either part may be empty"""
if p[1:2] == ':':
return p[0:2], p[2:]
return '', p
def split(self, p):
"""Split a pathname.
Return tuple (head, tail) where tail is everything after the final slash.
Either part may be empty."""
d, p = self._splitdrive(p)
# set i to index beyond p's last slash
i = len(p)
while i and p[i - 1] not in '/\\':
i = i - 1
head, tail = p[:i], p[i:] # now tail has no slashes
# remove trailing slashes from head, unless it's all slashes
head2 = head
while head2 and head2[-1] in '/\\':
head2 = head2[:-1]
head = head2 or head
return d + head, tail
def _openNewFile(self, fName):
# arg maybe include new csv file
if fName:
tempCsv = fName + "_Latencies_data.csv"
else:
tempCsv = "Command_Latencies_data.csv"
if self.logFilePath.endswith('.log'):
subPath2, _ = self.split(self.logFilePath)
subPath1, _ = self.split(subPath2)
filePath, _ = self.split(subPath1)
else:
filePath = self.logFilePath
fileName = os.path.join(filePath, tempCsv)
if os.path.exists(fileName):
self.logger.info("-------file exist--------")
return fileName
def _calSTD(self, sortedRawList=None, avgTime=0):
n = len(sortedRawList)
if n == 1:
return 0
tempList = [pow((item - avgTime), 2) for item in sortedRawList]
stdDeviation = math.sqrt(sum(tempList) / float(n - 1))
return stdDeviation
def classOfNines(self, data, outliers=100):
"""
For quality of service (QoS) we usually need to print not only the maximum but also the 99% largest element
:param data: All data points collected, probably latencies
:type data: list
:param outliers: Have at least this many outliers before calculating that class of nines
:type outliers: int
:return results: The class of nines results
:rtype results: dict
Example
results = classOfNines(range(1, 10000000+1))
print [(key, results[key]) for key in sorted(results.keys())]
"""
data.sort()
results = {}
nines = [0.99, 0.999, 0.9999, 0.99999, 0.999999, 0.9999999, 0.99999999, 0.999999999, 0.9999999999]
n = 0
listLen = len(data)
while True:
if not listLen / 10 ** (n + 2) >= outliers:
break
index = int(nines[n] * listLen) - 1
results[nines[n]] = data[index]
n += 1
return results
def _calAllData(self, cmdType, cmdName, rawData, sampleNumber=None):
"""
Statistics of raw latency data
:param cmdType: Category of the operation, eg, QoS, SCSI, UPIU
:param cmdName: Details of the cmd, eg, 4K read dirty
:param rawData: raw latency data
:param sampleNumber: the count of data points collected
:return: statistics data list, eg, [['QoS', '4K read dirty', 'Min', 20], ['QoS', '4K read dirty', 'Max', 30]...]
"""
rawDataList = copy.deepcopy(rawData)
if not sampleNumber:
sampleNumber = len(rawData)
rawDataList.sort()
if len(rawDataList) == 1:
avgTime = rawDataList[0]
maxTime = rawDataList[0]
minTime = rawDataList[0]
median = rawDataList[0]
stdDeviation = 0 # standard deviation
else:
avgTime = float(sum(rawDataList)) / len(rawDataList)
maxTime = max(rawDataList)
minTime = min(rawDataList)
sortedTimes = sorted(rawDataList)
if (len(sortedTimes) == 0):
median = 0
else:
if (len(sortedTimes) % 2) == 1:
median = sortedTimes[len(sortedTimes) / 2]
else:
median = (sortedTimes[(len(sortedTimes) / 2) - 1] + sortedTimes[(len(sortedTimes) / 2)]) / 2
stdDeviation = self._calSTD(rawDataList, avgTime)
keys = []
values = []
countNum = 0
nines = self.classOfNines(rawDataList)
nines1 = sorted(nines.iteritems(), key=lambda d: d[1], reverse=False)
if len(rawDataList) < 10000:
values = ["NA", "NA", "NA"]
else:
for key, value in nines1:
keys.append(key)
values.append(value)
countNum = countNum + 1
if countNum == 3:
break
if len(keys) == 1:
keys.extend(["NA","NA"])
values.extend(["NA","NA"])
if len(keys) == 2:
keys.append("NA")
values.append("NA")
allData = [
[cmdType] + [cmdName] + ['Min'] + [minTime],
[cmdType] + [cmdName] + ['Avg'] + [avgTime],
[cmdType] + [cmdName] + ['Median'] + [median],
[cmdType] + [cmdName] + ['Std Dev'] + [stdDeviation],
[cmdType] + [cmdName] + ['99%'] + [values[0]],
[cmdType] + [cmdName] + ['99.9%'] + [values[1]],
[cmdType] + [cmdName] + ['99.99%'] + [values[2]],
[cmdType] + [cmdName] + ['Max'] + [maxTime],
[cmdType] + [cmdName] + ['sampleNumber'] + [sampleNumber]
]
# allData = [cmdType] + [CMD] + [minTime, avgTime, median, stdDeviation] + values + [maxTime, sampleNumber]
self.logger.info("-----------------------------------------------------------------------------------------------------------")
self.logger.info("{:<16} {:<48} {:<8} {:<16} {:<8} {:<16} {:<8} {:<8} {:<8} {:<8} {:<8}".
format("cmdType", "CMD", "MinTime", "AVERAGE", "Median", "STD", "99%", "99.9%", "99.99%",
"MaxTime", "sampleNumber"))
self.logger.info("{:<16} {:<48} {:<8} {:<16} {:<8} {:<16} {:<8} {:<8} {:<8} {:<8} {:<8}".
format(cmdType, cmdName, minTime, avgTime, median, stdDeviation, values[0],
values[1], values[2], maxTime, sampleNumber))
self.logger.info("------------------------------------------------------------------------------------------------------------")
return allData
def _handleLatencies(self, allData, fName=None):
# allData---type: list or int(no raw data)
# file name---type str without(.csv):new csv file default:
fileName = self._openNewFile(fName)
wrFileHdlr = open(fileName, "ab+")
wt = csv.writer(wrFileHdlr)
for i in xrange(len(allData)):
data = allData[i]
wt.writerow(data)
wrFileHdlr.close()
def _saveRawData(self, nameList, rawDataList):
"""
Save raw latency data to csv file
:param nameList: a list which contains the cm names, used in csv file naming
:param rawDataList: raw data corresponding to nameList
:return: None
"""
for i in range(len(nameList)):
if self.logFilePath.endswith('.log'):
rawDataFile = self.logFilePath.rstrip('.log') + "_" + nameList[i] + "_raw_data.csv"
else:
rawDataFile = self.logFilePath + "/" + self.logFileName[:-4] + "_" + nameList[i] + "_raw_data.csv"
wrFileHdlr = open(rawDataFile, "a+")
for execTime in rawDataList[i]:
wrFileHdlr.write("{}\n".format(execTime))
wrFileHdlr.close()
def _rawDataFileName(self, name):
"""
Get target raw data file, related to func _saveRawData()
:param name: cmd name
:return: absolute raw data file path
"""
if self.logFilePath.endswith('.log'):
rawDataFile = self.logFilePath.rstrip('.log') + "_" + name + "_raw_data.csv"
else:
rawDataFile = self.logFilePath + "/" + self.logFileName[:-4] + "_" + name + "_raw_data.csv"
return rawDataFile
def _nandTemperaturePrint(self):
"""Print the current NAND temperature in Celsius
"""
temp = self.lib.vuHighLevelLib.getNandTemperature()
self.logger.info(temp)
def _generateCommandIndex(self, startLba=0, endLba=0, chunkSize=1, randCount=100000, percentage=50):
# random address list without overlap
def randrange(start, end, size, randCount):
if ((end - start) < (size - 1)) or (len(lbaList) == randCount):
return 0
lba = random.randint(start, end - size + 1)
lbaList.append(lba)
randrange(start, lba - 1, size, randCount)
randrange(lba + size, end, size, randCount)
# Make sure keep more than 50% UDA after unmap. if necessary, should rerun the test to collect 100,000
UDAPerct = (endLba - startLba + 1) * percentage / 100
gpNum = UDAPerct / chunkSize
sampleCnt = gpNum if gpNum < randCount else randCount
if chunkSize == 1:
cmdIndex = [[tempLba, tempLba + chunkSize - 1] for tempLba in
random.sample(xrange(startLba, endLba + 1 - chunkSize + 1), sampleCnt)]
else:
lbaList = []
randrange(startLba, endLba, chunkSize, randCount)
cmdIndex = [[tempLba, tempLba + chunkSize - 1] for tempLba in random.sample(lbaList, sampleCnt)]
return cmdIndex
def _getUnmapTimeQD(self, unmapCMDIndex=None, unmapSize=0x1, queueDepth=8):
responseList = []
unmapLatencies = []
outstandingCmd = {}
outstandingCmdTotalCnt = 0
while (unmapCMDIndex):
if (len(outstandingCmd) < queueDepth):
currentCmd = unmapCMDIndex.pop(0)
startLba = currentCmd[0]
usr = self.lun[self.lunID].UnmapSimple(startLba, unmapSize)
outstandingCmdTotalCnt += 1
outstandingCmd[usr.SequenceId] = ('Unmap', 0, currentCmd[0], unmapSize)
else:
t = timer(30)
while (not t.expired) and (len(responseList) == 0):
try:
responseList = self.lun[self.lunID].CompletedResponses
except Exception as e:
self.testFailed("Exception occurred: %s" % e)
if t.expired:
self.testFailed("Timeout occurred after %s,no responses returned " % t)
for resp in responseList:
execTime = float(resp.ExecTime * 10 ** -3)
sequenceId = resp.SequenceId
unmapLatencies.append(execTime)
outstandingCmd.pop(sequenceId)
# cmdType, buffNum, randLba, tranfLen = outstandingCmd.pop(sequenceId)
# # For fw request, add vu to get command's ingo
# if self.vuLatencyEn and not self.competitor:
# cmdInfo = (cmdType, self.lunID, randLba, tranfLen, resp.TaskTag, sequenceId, execTime)
# self._vuLatency(cmdInfo)
responseList = []
while len(outstandingCmd):
try:
responseList = self.lun[self.lunID].WaitForAllResponses()
except Exception as e:
self.testFailed("Exception occurred: %s" % e)
for resp in responseList:
execTime = float(resp.ExecTime * 10 ** -3)
sequenceId = resp.SequenceId
unmapLatencies.append(execTime)
cmdType, buffNum, randLba, tranfLen = outstandingCmd.pop(sequenceId)
# For fw request, add vu to get command's info
# if self.vuLatencyEn and not self.competitor:
# cmdInfo = (cmdType, self.lunID, randLba, tranfLen, resp.TaskTag, sequenceId, execTime)
# self._vuLatency(cmdInfo)
#
# # If dump the remaining cache data.
# if self.vuLatencyEn and not self.competitor and self.dumpCache and len(self.vuCacheDict):
# self.logger.info("The last vcCacheDice length:{}".format(len(self.vuCacheDict)))
# self._dumpCacheData()
return unmapLatencies
def _collectUnmapLatencies(self, unmapSize=0x1, QD=8, totalCnt=100000, maxUnmapPert=100):
unmapLatenciesList = []
unmapCMDIndex = self._generateCommandIndex(0, self.maxLba, chunkSize=unmapSize, randCount=totalCnt,
percentage=maxUnmapPert)
unmapLatenciesList.extend(self._getUnmapTimeQD(unmapCMDIndex, unmapSize, QD))
while (len(unmapLatenciesList) < totalCnt):
self.logger.info('{} sampling points collected, switch to next iteration'.format(len(unmapLatenciesList)))
self._preConditionForSAS()
unmapCMDIndex = self._generateCommandIndex(0, self.maxLba, chunkSize=unmapSize,
randCount=(totalCnt - len(unmapLatenciesList)))
unmapLatenciesList.extend(self._getUnmapTimeQD(unmapCMDIndex, unmapSize, QD))
return unmapLatenciesList
def _getVariables(self, lunID):
"""
Return the variable needed for this script to run. This method is overwritten by children classes.
:param lunID: (int) Current LUN
:returns cmdCalls ([functions]) The commands that the latencies are to be taken from
:returns cmdNames ([str]) Names of the commands in cmdCalls. Set to None if you don't want latencies from that command.
:returns isSequential (bool) When looping through should commands be run in order like 111122223333 (True) or 123123123123 (False)
"""
def buffer_():
return self.mist.Buffer(sectorCount=1, bytesPerSector=LBA_SIZE)
def inquiry():
return self.lun[lunID].InquirySync(enableVitalProductData=True, pageCode=0, buf=buffer_())
cmdCalls = [inquiry]
cmdNames = ["Inquiry"]
isSequential = True
getVariables = namedtuple("getVariables", "cmdCalls cmdNames isSequential")
return getVariables(cmdCalls=cmdCalls, cmdNames=cmdNames, isSequential=isSequential)
def _enterAndExitH8(self, delay=0):
H8_Exit_lat = []
respHibern8Enter = self.lib.dme.hibernateEnter()
if (respHibern8Enter != 0):
self.addFailure("Hibern8 enter failed. status is {} ".format(respHibern8Enter))
self.logger.info('Step2.1-----Wait 5s, and monitor Iccq2---')
time.sleep(delay * 1.0 / 1000)
# self.lib.powerMeasurement.measureVccCurrent(1000, delay=1)
if delay == 0:
one_million_samples = 1 * 1000 * 1000
self.lib.powerMeasurement.measureVccqVccq2Current(samples=2 * one_million_samples, sampleRate=250, test="Idle",
temperature="25C", hibernate=True)
self.logger.info("Step2.2-----Exit hibernate -------")
response = self.lun[self.lun.keys()[0]].Dme.HibernateExit()
if ((response.Argument2 & 0xFF) != 0):
self.addFailure("Hibern8 Exit DME command failed. error code is {} ".format(response.Argument2))
else:
self.logger.info("exec time of exit hibernate is {}us".format(response.ExecTime))
H8_Exit_lat.append(response.ExecTime)
def getBigLunId(self):
lunList = self.lib.reportLuns.getAvailableNormalLuns()
bigLun = 0
maxLbaBigLun = 0
for lunId in lunList:
buf = self.mist.Buffer(1, 8)
tempLba = self.lun[lunId].ReadCapacity10Sync(buf).MaxLba
if tempLba > maxLbaBigLun:
bigLun = lunId
maxLbaBigLun = tempLba
maxUnmapLbaSize = self.lib.unmapAsync._getMaxUnmapLbaSize(bigLun)
intCapacityGB = maxLbaBigLun / self.lbaStepSize
maxLba = intCapacityGB * self.lbaStepSize
lbaMainStart = 0
LbaRange = [(startLba, endLba) for startLba, endLba in
zip(range(lbaMainStart, maxLba - self.lbaStepSize, self.lbaStepSize), range(lbaMainStart + self.lbaStepSize - 1, maxLba, self.lbaStepSize))]
return bigLun, maxLbaBigLun, LbaRange, maxUnmapLbaSize
def getLatencies(self, cmdCalls, isSequential, secIdleBoundary):
usLatencies = self.lib.performanceAdHoc.loopCmd(
command=cmdCalls, loopSize=self.totalCmdNum, sequential=isSequential, idleSleep=secIdleBoundary, verbose=self.verbose)
return usLatencies
def Test(self):
"""
This is the main test function of the TC Script.
"""
self._eraseFullCardPost()
self.logger.info("<Step1> PRECONDITION: Lun config")
self._newTargetConfg()
self.lunID, self.maxLba, self.LbaRange, self.maxUnmapLbaSize = self.getBigLunId()
self.vuCacheDict = collections.OrderedDict()
self.vuLatencyTO = self.vuCMDTO
self.logger.info("Lun {}".format(self.lunID))
# ####### VARIABLES ####### #
cmdCalls, cmdNames, isSequential = self._getVariables(self.lunID)
self.cmdNames = cmdNames
# ####### PRECONDITION ####### #
self.logger.info("<Step2> Send 1000 * TUR to measure host overhead ")
oldValues, avgTeUnRe, idleBoundary = self._preCondition(lunID=self.lunID)
secIdleBoundary = self.lib.generic.changeUnits("seconds", self.outputUnit, idleBoundary)
# ####### MAIN ####### #
self.logger.info("<Step3> Send {} * {} commands to measure latency".format(self.totalCmdNum, cmdNames))
usLatencies = self.getLatencies(cmdCalls, isSequential, secIdleBoundary)
latencies = self._afterLoopBeforeAveraging(usLatencies=usLatencies)
self._saveRawData(self.cmdNames, latencies)
# make sure to use len(latencies) in case of duplicated cmdNames
for i in xrange(len(latencies)):
allData = self._calAllData(self.type, cmdNames[i], latencies[i])
self._handleLatencies(allData, self.logFileName[:-4])
self.logger.debug("POSTCONDITION")
self._postCondition(lunID=self.lunID, oldValues=oldValues)
self._eraseFullCardPost()
def Main():
test = Latency_00_Command_Base()
exit(test.Run())
if __name__ == "__main__":
Main()
[源代码打包下载]
Latency.rar(50积分)[6 次下载]