AI时代，找源码已成为过去式，纪念我过去的十年0(回)
69天前
还有人吗2(回)
87天前会python真的可以为所欲为0(回)
567天前
这里还有人吗2(回)
752天前这里还有人吗0(回)
752天前
每天面对着电脑屏幕，敲打键盘。我所面对的并不只是代码，而是一种生活方式。0(回)
977天前
Latency.rar(免费下载)[11 次下载]import random |
import time |
import EnvSetup.EnvSetup |
import Library.UFS.TestInstance as TestInstance |
# import config.UFS |
import os |
import copy |
import math |
import csv |
from Library.UFS.ufsDefines import * |
from Library.Generic import timer |
# Extra Imports |
from collections import namedtuple |
import collections |
class Latency_00_Command_Base(TestInstance.TestInstance): |
SIZE_1KB = 1024 |
SIZE_1MB = 1024 * SIZE_1KB |
SIZE_1GB = 1024 * SIZE_1MB |
LBA_SIZE = 4 * SIZE_1KB |
lbaStepSize = SIZE_1GB / LBA_SIZE |
DEBUG = False |
vcc = -1 |
vccq = -1 |
vccq2 = -1 |
atariCategory = 'Latency' |
atariGroup = 'TSM' |
outputUnit = 'ms' |
pwrMode = 'Active' |
exitOnFailure = False |
verbose = False |
type = 'Scsi' |
def AddCommandArgs(self): |
super(Latency_00_Command_Base, self).AddCommandArgs() |
# configure divice |
self.parser.add_argument('--mediaLunCount', type=int, default=1, help="""configure device with defalut(0ne) lun""") |
self.parser.add_argument('--bootLunCount', type=int, default=0, help="""configure device with default(default 0) boot Lun""") |
self.parser.add_argument('--competitor', type=bool, default=False, help="""whether the device comes from competitor. defalut False""") |
self.parser.add_argument('--tgtProvisionType', type=int, default=3, help="""provisioning type. discard,2,TPRZ=0/erase,3,TPRZ=1""") |
# argument for precondition |
self.parser.add_argument('--seqTransferTL', type=int, default=0x80, help="""sequential write TL, default 512K""") |
self.parser.add_argument('--seqData', type=int, default=0xAA, help="""sequential Write Data""") |
self.parser.add_argument('--randNumber', type=int, default=100000, help="""random write cmd count per GB""") |
self.parser.add_argument('--randData', type=int, default=0x55, help="""random write data""") |
# argument for data collection and flow contorl |
self.parser.add_argument('--basicCmdNum', type=int, default=100000, help="""command count for each iteration/GB""") |
self.parser.add_argument('--totalCmdNum', type=int, default=100000, help="""Sample points needed for data collection""") |
# argument for debug function |
# self.parser.add_argument('--specialSaveRawData', type=bool, default=False, help="""For special format save raw data collection""") |
self.parser.add_argument('--vuLatencyEn', type=bool, default=False, help="""If using latency VU, should use latency FW. print data profiling debug.""") |
self.parser.add_argument('--dumpCache', type=bool, default=False, help="""If necessary, open this. each cache count, script will print command info and vu data""") |
self.parser.add_argument('--vuCMDTO', type=float, default=30.0, help="""For read/sync/other command time out parameter. if command > 30ms, timeout""") |
self.parser.add_argument('--vuWriteTO', type=float, default=30.0, help="""Just for write, if command > 30ms, timeout""") |
self.parser.add_argument('--cacheCount', type=int, default=8, help="""save how many command into cache.""") |
# Others |
self.parser.add_argument('--defrageTable', type=bool, default=False, help="""dirty card or not""") |
def _reconfigDevice(self, bSecureRemovalTypeSet=0, bProvisionTypeReset=3): |
if not self.competitor: |
self.vuLib.vuCmdtranslate.restoreWriteOnce() |
response = self.lun[0].Query.ReadDescriptorSync(0x1, 0, 0, 0x90) |
reconfigBuffer = response.DataSegmentBuffer |
if reconfigBuffer.GetByte(2) != 0: |
reconfigBuffer.SetByte(2, 0) |
reconfigBuffer.SetByte(7, bSecureRemovalTypeSet) |
self.lun[0].Query.WriteDescriptorSync(0x1, 0, 0, 0x90, reconfigBuffer) |
maxAllocUinit = self.lib.query.getdEnhanced1MaxNAllocU() |
HighByte = (int(maxAllocUinit) & 0xFF00) >> 8 |
LowByte = int(maxAllocUinit) & 0xFF |
response = self.lun[0].Query.ReadDescriptorSync(1, 0, 0, 0x90) |
ReconfigBuf = response.DataSegmentBuffer |
for i in range(1, 9): |
ReconfigBuf.SetByte(i * 16 + 0xA, bProvisionTypeReset) |
ReconfigBuf.SetByte(i * 16, 0) |
ReconfigBuf.SetByte(i * 16 + 1, 0) |
ReconfigBuf.SetByte(i * 16 + 6, 0) |
ReconfigBuf.SetByte(i * 16 + 7, 0) |
ReconfigBuf.SetByte(0x16, HighByte) |
ReconfigBuf.SetByte(0x17, LowByte) |
ReconfigBuf.SetByte(0x10, 1) |
ReconfigBuf.SetByte(0x2, 0) |
self.lun[0].Query.WriteDescriptorSync(1, 0, 0, 0x90, ReconfigBuf) |
def _doBKOPs(self, defautTime=400): |
buf = self.mist.Buffer(0x1, LBA_SIZE) |
t = timer(defautTime) |
while not t.expired: |
if self.lib.query.getbBackgroundOpStatus() > 0: |
wExceptionEventStatus = self.lib.query.getwExceptionEventStatus() |
self.logger.info("send one read command") |
self.lun[self.lunID].Read10SimpleSync(0x1, 0x1, buf) |
self.logger.info("--------Get wExceptionEventStatus:{}".format(wExceptionEventStatus & 0x4)) |
time.sleep(60) |
else: |
break |
currentBKOPs = self.lib.query.getbBackgroundOpStatus() |
if currentBKOPs == 0 or currentBKOPs == 1: |
self.logger.info("current BKOPs has been completed") |
else: |
self.testFailed('bBackgroundOpStatus is still {} after {}'.format(currentBKOPs, defautTime)) |
def _newTargetConfg(self, bootOrNot=False): |
if not self.competitor: |
self.logger.info("########## Create new target configuration descriptor ##########") |
if bootOrNot: |
bootCapacity = 16 |
self.lib.lunConfig.createTargetConfig(mediaLunCount=self.mediaLunCount, bootLunCount=self.bootLunCount, setBigLun=True, |
useAllCapacity=True, setHighPriorityLun=False, |
bProvisioningType=self.tgtProvisionType, bootCapacity_MB=bootCapacity) |
else: |
self.lib.lunConfig.createTargetConfig(mediaLunCount=self.mediaLunCount, bootLunCount=self.bootLunCount, setBigLun=True, |
useAllCapacity=True, setHighPriorityLun=False, bProvisioningType=self.tgtProvisionType) |
self.lib.lunConfig.printDescriptor() # Print the new created target descriptor |
disabledLun = self.lib.lunConfig.getDisableLunInTargetConfig() |
self.logger.info("Disabled LUs are: %s" % disabledLun) |
targetCfgOutline = self.lib.lunConfig.getTargetConfigOutline() |
for key, value in targetCfgOutline.items(): |
self.logger.info("Category {} : {}".format(key, value)) |
# Write the new target configuration descriptor to device |
self.lib.lunConfig.writeTargetConfigToDevice() |
else: |
self.logger.info("the unit comes from competitor") |
def _latencyOfPurge(self, purgeTimeOut=None): |
""" |
Original purgeTimeoOut=4000 |
Purge all unmaped LBA(s) |
NOTE: This latency is in increments of 1ms |
:param purgeTimeOut: (int) in seconds the timeout period to wait for the purge command |
:return usPurgeTime: (float) command execution time in microseconds (us) |
""" |
self.logger.info("***********************do purge*********************") |
if purgeTimeOut is None: |
purgeTimeOut = self.cfg.timeout.purgeTimeout |
# Just for leo, to pass this case. |
if self.product == "Leo": |
purgeTimeOut = 10000 |
setPurgeResp = self.lib.query.setfPurgeEnable() |
if setPurgeResp == 1: |
purgeStartTime = time.time() |
else: |
self.testFailed( |
"Set fPurgeEnable flag enable fail, and the fPurgeEnable flag value is {}".format(setPurgeResp)) |
bPurgeStatus = -1 |
while bPurgeStatus not in (PURGE_STATUS_STOPPED_BY_HOST, PURGE_STATUS_COMPLETED): |
bPurgeStatus = self.lib.query.getbPurgeStatus() |
if time.time() - purgeStartTime > purgeTimeOut: |
self.testFailed("FAIL bPurgeStatus ({}) not set to {} before timeOut {}".format(setPurgeResp, ( |
PURGE_STATUS_STOPPED_BY_HOST, PURGE_STATUS_COMPLETED), purgeTimeOut)) |
purgeEndTime = time.time() |
secondsPurgeTime = purgeEndTime - purgeStartTime |
usPurgeTime = self.lib.generic.changeUnits("us", "seconds", secondsPurgeTime) |
return usPurgeTime |
def _eraseFullCardPost(self): |
self.logger.info("pre/post condition: format unit and purge") |
lunList = self.lib.reportLuns.getAvailableNormalLuns() |
for lunID in lunList: |
self.lun[lunID].FormatUnitSimpleSync() |
self._latencyOfPurge() |
def _defragTableVB(self): |
tl = 1 |
buf = self.mist.Buffer(1, 8) |
maxLba = self.lun[self.lunID].ReadCapacity10Sync(buf).MaxLba |
writeBuffer = self.mist.Buffer(tl, LBA_SIZE) |
writeBuffer.Fill(0x55) |
for i in range(12): |
startLba = random.randint(0, 1020) |
while startLba < maxLba: |
if startLba + tl > maxLba: |
startLba = maxLba - tl |
self.lun[self.lunID].Write10Sync(lba=startLba, count=tl, disablePageOut=False, fua=False, fuaNv=False, buf=writeBuffer) |
startLba = startLba + 1024 |
def _sendWriteCMD(self, startLba, endLba, transferLen=1, writeData=0xAA, cmdNumber=0, seq=False, QD=32): |
outstandingCmd = {} |
responseList = [] |
timeout = 60 |
buffers = [self.mist.Buffer(transferLen, LBA_SIZE) for _ in range(QD)] |
buffTrk = range(QD) |
temp = 0 |
if seq: |
self.logger.debug("sequential write from startLba=--{}-- to endLba=--{}--".format(startLba, endLba)) |
while (startLba <= endLba): |
if startLba + transferLen > endLba: |
tempTL = endLba - startLba + 1 |
else: |
tempTL = transferLen |
if len(outstandingCmd) < QD: |
buffNum = buffTrk.pop() |
buf = buffers[buffNum] |
buf.Fill(writeData) |
usr = self.lun[self.lunID].Write10Simple(startLba, tempTL, buf) |
outstandingCmd[usr.SequenceId] = (buffNum) |
if len(outstandingCmd) == QD: |
startTime = time.time() |
while len(responseList) == 0: |
elapsedTime = time.time() - startTime |
if elapsedTime > timeout: |
break |
try: |
responseList = self.lun[self.lunID].CompletedResponses |
except self.mist.ufs.UpiuCommandError as e: |
self.testFailed("UpiuCommandError happened {}".format(e)) |
except Exception as e: |
self.testFailed("Exception happened {}".format(e)) |
for resp in responseList: |
sequenceId = resp.SequenceId |
buffNum = outstandingCmd.pop(sequenceId) |
buffTrk.append(buffNum) |
responseList = [] |
startLba += tempTL |
if startLba == endLba + 1: |
if 0 < len(outstandingCmd) < QD: |
for i in range(len(outstandingCmd)): |
self.lun[self.lunID].WaitForOneResponse() |
else: |
if cmdNumber: |
self.logger.debug("random send cmdNumber={} write command in startLba={},endLba={}".format(cmdNumber, startLba, endLba)) |
while (temp < cmdNumber): |
if len(outstandingCmd) < QD: |
buffNum = buffTrk.pop() |
buf = buffers[buffNum] |
buf.Fill(writeData) |
tempLba = random.randint(startLba, endLba - transferLen + 1) |
usr = self.lun[self.lunID].Write10Simple(tempLba, transferLen, buf) |
temp = temp + 1 |
outstandingCmd[usr.SequenceId] = (buffNum) |
if len(outstandingCmd) == QD: |
startTime = time.time() |
while len(responseList) == 0: |
elapsedTime = time.time() - startTime |
if (elapsedTime > timeout) and len(responseList) == 0: |
self.testFailed("Timeout occurred after %s,no responses returned " % elapsedTime) |
break |
try: |
responseList = self.lun[self.lunID].CompletedResponses |
except self.mist.ufs.UpiuCommandError as e: |
self.testFailed("UpiuCommandError happened {}".format(e)) |
except Exception as e: |
self.testFailed("Exception happened {}".format(e)) |
for resp in responseList: |
sequenceId = resp.SequenceId |
buffNum = outstandingCmd.pop(sequenceId) |
buffTrk.append(buffNum) |
responseList = [] |
if temp == cmdNumber: |
for i in range(len(outstandingCmd)): |
self.lun[self.lunID].WaitForOneResponse() |
else: |
writeBuffer = self.mist.Buffer(transferLen, LBA_SIZE) |
writeBuffer.Fill(writeData) |
self.logger.debug("random send write command on current lbaRange until all LBA has been written") |
self.lib.writeReadAsync.writeRandom(self.lunID, "Write10", writeBuffer, startLba, endLba, transferLen, QD, fua=True) |
def _overwriteCli(self): |
self.responseLoggerIsEnabled = False |
self.responseLoggerReportUpiu = False |
self.responseLoggerReportFailOnly = True |
self.needstandardTestStart = False |
def _preConditionForSAS(self, SSPreCondition=False): |
if self.DEBUG: |
self.logger.info('DEBUG running, precondition skipped') |
return |
self.logger.info("sequential write full card with ChunkSize *** 512KB ***, data is 0xAA") |
buff = self.mist.Buffer(1, 8) |
endLba = self.lun[self.lunID].ReadCapacity10Sync(buff).MaxLba |
self._sendWriteCMD(0, endLba, transferLen=self.seqTransferTL, seq=True) |
if not SSPreCondition: |
self.logger.info("============Random write 100000 per GB with pattern 0x55============") |
for startLba, endLba in self.LbaRange: |
self._sendWriteCMD(startLba, endLba, writeData=self.randData, cmdNumber=self.randNumber) |
else: |
self.logger.info("precondition for SS's test method") |
cmdNumber = self.maxLba / self.seqTransferTL |
self._sendWriteCMD(0, self.maxLba, transferLen=self.seqTransferTL, writeData=self.randData, cmdNumber=cmdNumber) |
if self.defrageTable: |
self._defragTableVB() |
def _preCondition(self, lunID, loopSize=1000): |
""" |
Precondition options you can overwrite by children classes. |
:param lunID: current lun |
:return oldValues: (dict) the values before this method changed them |
:return avgTeUnRe: (float) average test unit ready time |
:return secIdleBoundary: (float) time to put device in idle mode in seconds |
""" |
newValues = {} |
if hasattr(self, 'newWce') and self.newWce >= 0: |
newValues["newWce"] = self.newWce |
if hasattr(self, 'newRcd') and self.newRcd >= 0: |
newValues["newRcd"] = self.newRcd |
if hasattr(self, 'gearHS') and self.gearHS >= 0: |
newValues["gearHS"] = self.gearHS |
if hasattr(self, 'rateHS') and self.rateHS != "None": |
newValues["rateHS"] = self.rateHS |
if hasattr(self, 'ufsPwrMode') and self.powerMode >= 0: |
newValues["ufsPwrMode"] = self.ufsPwrMode |
if hasattr(self, 'laneNum') and self.laneNum >= 0: |
newValues["laneNum"] = self.laneNum |
if hasattr(self, 'bDataReliability') and self.bDataReliability >= 0: |
newValues["bDataReliability"] = self.bDataReliability |
newValues["newVoltages"] = self.lib.performanceAdHoc.getVoltages( |
vcc=self.vcc, vccq=self.vccq, vccq2=self.vccq2) |
oldValues = self.lib.performanceAdHoc.latencyPreCondition(lunID=lunID, newValues=dict(newValues), verbose=False, |
exitOnFailure=self.exitOnFailure) |
self.logger.parameter(paramName="newValues", value=newValues, category=self.atariCategory, group=self.atariGroup) |
self.logger.parameter(paramName="oldValues", value=oldValues, category=self.atariCategory, group=self.atariGroup) |
(usAvgTeUnRe, _, _) = self.lib.performanceAdHoc.testUnitReadyLoopAndPrint(lunID=lunID, loopSize=loopSize, verbose=False) |
avgTeUnRe = self.lib.generic.changeUnits(self.outputUnit, "us", usAvgTeUnRe) |
self.logger.parameter(paramName="Test Unit Ready Average", value=avgTeUnRe, unit=self.outputUnit, category=self.atariCategory, group=self.atariGroup) |
if self.pwrMode == "Idle": |
(_, rawBoundary, rawUnit) = self.cfg.latency.getLatencyExpected("Typical", "Active to Idle") |
idleBoundary = self.lib.generic.changeUnits(self.outputUnit, rawUnit, rawBoundary) |
else: |
idleBoundary = None |
preCondition = namedtuple("preCondition", "oldValues avgTeUnRe secIdleBoundary") |
return preCondition(oldValues=oldValues, avgTeUnRe=avgTeUnRe, secIdleBoundary=idleBoundary) |
def _dumpCacheData(self): |
self.logger.info("===================Dump CMD/VU Buffer=======================") |
self.logger.info("NOTE: CMD Info Format: (cmdType, lun, lba, transferLen, taskTag, sequenceId, exec(ms)") |
self.logger.info("Debug. cache cmd count:{}".format(len(self.vuCacheDict))) |
for cmdKey, vuValue in self.vuCacheDict.items(): |
tempList = [] |
tempList.append(cmdKey) |
self.logger.info("cmdKey(ms):{}".format(cmdKey)) |
self.logger.info("vuValue:{}".format(vuValue)) |
for index, tempValue in enumerate(vuValue): |
tempList.append(tempValue) |
self._saveRawData(["TimeOut_VU_Get_Latency"], [tempList]) |
# if the whole cache data are dumped, clear cache. |
self.vuCacheDict.clear() |
def _vuLatency(self, cmdInfo, checkTO=True): |
# cmdInfo = (cmdType, lun, lba, transferLen, taskTag, sequenceId, execTime) |
cmdType = cmdInfo[0] |
execTime = cmdInfo[-1] |
if self.cacheCount: |
if len(self.vuCacheDict) >= self.cacheCount: |
if self.dumpCache: |
self._dumpCacheData() |
else: |
# if no dump data, then pop one item for the next cmd info. |
self.vuCacheDict.popitem(0) |
vuData = self.vuLib.vuCmdtranslate.getProfileDataLatency() |
self.vuCacheDict[cmdInfo] = vuData |
# Check command whether timeout.if yes, test failed immediately. |
if cmdType == 'Mode Sense': |
self.vuLatencyTO = 1 |
if cmdType == "Mode Select": |
self.vuLatencyTO = 100 |
self.logger.info("111----set command timeout {}ms for cmdType {}".format(self.vuLatencyTO, cmdType)) |
if checkTO and execTime >= self.vuLatencyTO: |
if self.cacheCount: |
self._dumpCacheData() |
try: |
vuData = self.vuLib.vuCmdtranslate.getProfileDataLatency() |
self.logger.info("Oops! Occurred cmd time out!, TargetTO:{} ms".format(self.vuLatencyTO)) |
self.logger.info("NOTE: CMD Info Format: (cmdType, lun, lba, transferLen, taskTag, sequenceId, exec(ms)") |
self.logger.info("CMD Info(ms):{}, ".format(cmdInfo)) |
self.testFailed("GetProfileData:{}".format(vuData)) |
except self.mist.ufs.UpiuCommandError as e: |
self.logger.info("Get profileData error:{}".format(e)) |
def _postCondition(self, lunID, oldValues): |
""" |
Postcondition options you can overwrite by children classes. |
:param lunID: current lun |
:param oldValues: (dict) the values to set the device back to before ending the next test |
""" |
self.lib.performanceAdHoc.latencyPostCondition(lunID=lunID, oldValues=oldValues) |
def _afterLoopBeforeAveraging(self, usLatencies): |
""" |
Some scripts need to modify the latencies before proceeding with the test |
:param lunID: (int) current lun |
:param usLatencies: [int] The latencies from the main function in micro seconds |
:returns latencies: [int] The modified latencies |
""" |
latencies = self.lib.generic.changeUnits(self.outputUnit, "us", usLatencies) |
return latencies |
def _splitdrive(self, p): |
"""Split a pathname into drive and path specifiers. Returns a 2-tuple |
"(drive,path)"; either part may be empty""" |
if p[1:2] == ':': |
return p[0:2], p[2:] |
return '', p |
def split(self, p): |
"""Split a pathname. |
Return tuple (head, tail) where tail is everything after the final slash. |
Either part may be empty.""" |
d, p = self._splitdrive(p) |
# set i to index beyond p's last slash |
i = len(p) |
while i and p[i - 1] not in '/\\': |
i = i - 1 |
head, tail = p[:i], p[i:] # now tail has no slashes |
# remove trailing slashes from head, unless it's all slashes |
head2 = head |
while head2 and head2[-1] in '/\\': |
head2 = head2[:-1] |
head = head2 or head |
return d + head, tail |
def _openNewFile(self, fName): |
# arg maybe include new csv file |
if fName: |
tempCsv = fName + "_Latencies_data.csv" |
else: |
tempCsv = "Command_Latencies_data.csv" |
if self.logFilePath.endswith('.log'): |
subPath2, _ = self.split(self.logFilePath) |
subPath1, _ = self.split(subPath2) |
filePath, _ = self.split(subPath1) |
else: |
filePath = self.logFilePath |
fileName = os.path.join(filePath, tempCsv) |
if os.path.exists(fileName): |
self.logger.info("-------file exist--------") |
return fileName |
def _calSTD(self, sortedRawList=None, avgTime=0): |
n = len(sortedRawList) |
if n == 1: |
return 0 |
tempList = [pow((item - avgTime), 2) for item in sortedRawList] |
stdDeviation = math.sqrt(sum(tempList) / float(n - 1)) |
return stdDeviation |
def classOfNines(self, data, outliers=100): |
""" |
For quality of service (QoS) we usually need to print not only the maximum but also the 99% largest element |
:param data: All data points collected, probably latencies |
:type data: list |
:param outliers: Have at least this many outliers before calculating that class of nines |
:type outliers: int |
:return results: The class of nines results |
:rtype results: dict |
Example |
results = classOfNines(range(1, 10000000+1)) |
print [(key, results[key]) for key in sorted(results.keys())] |
""" |
data.sort() |
results = {} |
nines = [0.99, 0.999, 0.9999, 0.99999, 0.999999, 0.9999999, 0.99999999, 0.999999999, 0.9999999999] |
n = 0 |
listLen = len(data) |
while True: |
if not listLen / 10 ** (n + 2) >= outliers: |
break |
index = int(nines[n] * listLen) - 1 |
results[nines[n]] = data[index] |
n += 1 |
return results |
def _calAllData(self, cmdType, cmdName, rawData, sampleNumber=None): |
""" |
Statistics of raw latency data |
:param cmdType: Category of the operation, eg, QoS, SCSI, UPIU |
:param cmdName: Details of the cmd, eg, 4K read dirty |
:param rawData: raw latency data |
:param sampleNumber: the count of data points collected |
:return: statistics data list, eg, [['QoS', '4K read dirty', 'Min', 20], ['QoS', '4K read dirty', 'Max', 30]...] |
""" |
rawDataList = copy.deepcopy(rawData) |
if not sampleNumber: |
sampleNumber = len(rawData) |
rawDataList.sort() |
if len(rawDataList) == 1: |
avgTime = rawDataList[0] |
maxTime = rawDataList[0] |
minTime = rawDataList[0] |
median = rawDataList[0] |
stdDeviation = 0 # standard deviation |
else: |
avgTime = float(sum(rawDataList)) / len(rawDataList) |
maxTime = max(rawDataList) |
minTime = min(rawDataList) |
sortedTimes = sorted(rawDataList) |
if (len(sortedTimes) == 0): |
median = 0 |
else: |
if (len(sortedTimes) % 2) == 1: |
median = sortedTimes[len(sortedTimes) / 2] |
else: |
median = (sortedTimes[(len(sortedTimes) / 2) - 1] + sortedTimes[(len(sortedTimes) / 2)]) / 2 |
stdDeviation = self._calSTD(rawDataList, avgTime) |
keys = [] |
values = [] |
countNum = 0 |
nines = self.classOfNines(rawDataList) |
nines1 = sorted(nines.iteritems(), key=lambda d: d[1], reverse=False) |
if len(rawDataList) < 10000: |
values = ["NA", "NA", "NA"] |
else: |
for key, value in nines1: |
keys.append(key) |
values.append(value) |
countNum = countNum + 1 |
if countNum == 3: |
break |
if len(keys) == 1: |
keys.extend(["NA","NA"]) |
values.extend(["NA","NA"]) |
if len(keys) == 2: |
keys.append("NA") |
values.append("NA") |
allData = [ |
[cmdType] + [cmdName] + ['Min'] + [minTime], |
[cmdType] + [cmdName] + ['Avg'] + [avgTime], |
[cmdType] + [cmdName] + ['Median'] + [median], |
[cmdType] + [cmdName] + ['Std Dev'] + [stdDeviation], |
[cmdType] + [cmdName] + ['99%'] + [values[0]], |
[cmdType] + [cmdName] + ['99.9%'] + [values[1]], |
[cmdType] + [cmdName] + ['99.99%'] + [values[2]], |
[cmdType] + [cmdName] + ['Max'] + [maxTime], |
[cmdType] + [cmdName] + ['sampleNumber'] + [sampleNumber] |
] |
# allData = [cmdType] + [CMD] + [minTime, avgTime, median, stdDeviation] + values + [maxTime, sampleNumber] |
self.logger.info("-----------------------------------------------------------------------------------------------------------") |
self.logger.info("{:<16} {:<48} {:<8} {:<16} {:<8} {:<16} {:<8} {:<8} {:<8} {:<8} {:<8}". |
format("cmdType", "CMD", "MinTime", "AVERAGE", "Median", "STD", "99%", "99.9%", "99.99%", |
"MaxTime", "sampleNumber")) |
self.logger.info("{:<16} {:<48} {:<8} {:<16} {:<8} {:<16} {:<8} {:<8} {:<8} {:<8} {:<8}". |
format(cmdType, cmdName, minTime, avgTime, median, stdDeviation, values[0], |
values[1], values[2], maxTime, sampleNumber)) |
self.logger.info("------------------------------------------------------------------------------------------------------------") |
return allData |
def _handleLatencies(self, allData, fName=None): |
# allData---type: list or int(no raw data) |
# file name---type str without(.csv):new csv file default: |
fileName = self._openNewFile(fName) |
wrFileHdlr = open(fileName, "ab+") |
wt = csv.writer(wrFileHdlr) |
for i in xrange(len(allData)): |
data = allData[i] |
wt.writerow(data) |
wrFileHdlr.close() |
def _saveRawData(self, nameList, rawDataList): |
""" |
Save raw latency data to csv file |
:param nameList: a list which contains the cm names, used in csv file naming |
:param rawDataList: raw data corresponding to nameList |
:return: None |
""" |
for i in range(len(nameList)): |
if self.logFilePath.endswith('.log'): |
rawDataFile = self.logFilePath.rstrip('.log') + "_" + nameList[i] + "_raw_data.csv" |
else: |
rawDataFile = self.logFilePath + "/" + self.logFileName[:-4] + "_" + nameList[i] + "_raw_data.csv" |
wrFileHdlr = open(rawDataFile, "a+") |
for execTime in rawDataList[i]: |
wrFileHdlr.write("{}\n".format(execTime)) |
wrFileHdlr.close() |
def _rawDataFileName(self, name): |
""" |
Get target raw data file, related to func _saveRawData() |
:param name: cmd name |
:return: absolute raw data file path |
""" |
if self.logFilePath.endswith('.log'): |
rawDataFile = self.logFilePath.rstrip('.log') + "_" + name + "_raw_data.csv" |
else: |
rawDataFile = self.logFilePath + "/" + self.logFileName[:-4] + "_" + name + "_raw_data.csv" |
return rawDataFile |
def _nandTemperaturePrint(self): |
"""Print the current NAND temperature in Celsius |
""" |
temp = self.lib.vuHighLevelLib.getNandTemperature() |
self.logger.info(temp) |
def _generateCommandIndex(self, startLba=0, endLba=0, chunkSize=1, randCount=100000, percentage=50): |
# random address list without overlap |
def randrange(start, end, size, randCount): |
if ((end - start) < (size - 1)) or (len(lbaList) == randCount): |
return 0 |
lba = random.randint(start, end - size + 1) |
lbaList.append(lba) |
randrange(start, lba - 1, size, randCount) |
randrange(lba + size, end, size, randCount) |
# Make sure keep more than 50% UDA after unmap. if necessary, should rerun the test to collect 100,000 |
UDAPerct = (endLba - startLba + 1) * percentage / 100 |
gpNum = UDAPerct / chunkSize |
sampleCnt = gpNum if gpNum < randCount else randCount |
if chunkSize == 1: |
cmdIndex = [[tempLba, tempLba + chunkSize - 1] for tempLba in |
random.sample(xrange(startLba, endLba + 1 - chunkSize + 1), sampleCnt)] |
else: |
lbaList = [] |
randrange(startLba, endLba, chunkSize, randCount) |
cmdIndex = [[tempLba, tempLba + chunkSize - 1] for tempLba in random.sample(lbaList, sampleCnt)] |
return cmdIndex |
def _getUnmapTimeQD(self, unmapCMDIndex=None, unmapSize=0x1, queueDepth=8): |
responseList = [] |
unmapLatencies = [] |
outstandingCmd = {} |
outstandingCmdTotalCnt = 0 |
while (unmapCMDIndex): |
if (len(outstandingCmd) < queueDepth): |
currentCmd = unmapCMDIndex.pop(0) |
startLba = currentCmd[0] |
usr = self.lun[self.lunID].UnmapSimple(startLba, unmapSize) |
outstandingCmdTotalCnt += 1 |
outstandingCmd[usr.SequenceId] = ('Unmap', 0, currentCmd[0], unmapSize) |
else: |
t = timer(30) |
while (not t.expired) and (len(responseList) == 0): |
try: |
responseList = self.lun[self.lunID].CompletedResponses |
except Exception as e: |
self.testFailed("Exception occurred: %s" % e) |
if t.expired: |
self.testFailed("Timeout occurred after %s,no responses returned " % t) |
for resp in responseList: |
execTime = float(resp.ExecTime * 10 ** -3) |
sequenceId = resp.SequenceId |
unmapLatencies.append(execTime) |
outstandingCmd.pop(sequenceId) |
# cmdType, buffNum, randLba, tranfLen = outstandingCmd.pop(sequenceId) |
# # For fw request, add vu to get command's ingo |
# if self.vuLatencyEn and not self.competitor: |
# cmdInfo = (cmdType, self.lunID, randLba, tranfLen, resp.TaskTag, sequenceId, execTime) |
# self._vuLatency(cmdInfo) |
responseList = [] |
while len(outstandingCmd): |
try: |
responseList = self.lun[self.lunID].WaitForAllResponses() |
except Exception as e: |
self.testFailed("Exception occurred: %s" % e) |
for resp in responseList: |
execTime = float(resp.ExecTime * 10 ** -3) |
sequenceId = resp.SequenceId |
unmapLatencies.append(execTime) |
cmdType, buffNum, randLba, tranfLen = outstandingCmd.pop(sequenceId) |
# For fw request, add vu to get command's info |
# if self.vuLatencyEn and not self.competitor: |
# cmdInfo = (cmdType, self.lunID, randLba, tranfLen, resp.TaskTag, sequenceId, execTime) |
# self._vuLatency(cmdInfo) |
# |
# # If dump the remaining cache data. |
# if self.vuLatencyEn and not self.competitor and self.dumpCache and len(self.vuCacheDict): |
# self.logger.info("The last vcCacheDice length:{}".format(len(self.vuCacheDict))) |
# self._dumpCacheData() |
return unmapLatencies |
def _collectUnmapLatencies(self, unmapSize=0x1, QD=8, totalCnt=100000, maxUnmapPert=100): |
unmapLatenciesList = [] |
unmapCMDIndex = self._generateCommandIndex(0, self.maxLba, chunkSize=unmapSize, randCount=totalCnt, |
percentage=maxUnmapPert) |
unmapLatenciesList.extend(self._getUnmapTimeQD(unmapCMDIndex, unmapSize, QD)) |
while (len(unmapLatenciesList) < totalCnt): |
self.logger.info('{} sampling points collected, switch to next iteration'.format(len(unmapLatenciesList))) |
self._preConditionForSAS() |
unmapCMDIndex = self._generateCommandIndex(0, self.maxLba, chunkSize=unmapSize, |
randCount=(totalCnt - len(unmapLatenciesList))) |
unmapLatenciesList.extend(self._getUnmapTimeQD(unmapCMDIndex, unmapSize, QD)) |
return unmapLatenciesList |
def _getVariables(self, lunID): |
""" |
Return the variable needed for this script to run. This method is overwritten by children classes. |
:param lunID: (int) Current LUN |
:returns cmdCalls ([functions]) The commands that the latencies are to be taken from |
:returns cmdNames ([str]) Names of the commands in cmdCalls. Set to None if you don't want latencies from that command. |
:returns isSequential (bool) When looping through should commands be run in order like 111122223333 (True) or 123123123123 (False) |
""" |
def buffer_(): |
return self.mist.Buffer(sectorCount=1, bytesPerSector=LBA_SIZE) |
def inquiry(): |
return self.lun[lunID].InquirySync(enableVitalProductData=True, pageCode=0, buf=buffer_()) |
cmdCalls = [inquiry] |
cmdNames = ["Inquiry"] |
isSequential = True |
getVariables = namedtuple("getVariables", "cmdCalls cmdNames isSequential") |
return getVariables(cmdCalls=cmdCalls, cmdNames=cmdNames, isSequential=isSequential) |
def _enterAndExitH8(self, delay=0): |
H8_Exit_lat = [] |
respHibern8Enter = self.lib.dme.hibernateEnter() |
if (respHibern8Enter != 0): |
self.addFailure("Hibern8 enter failed. status is {} ".format(respHibern8Enter)) |
self.logger.info('Step2.1-----Wait 5s, and monitor Iccq2---') |
time.sleep(delay * 1.0 / 1000) |
# self.lib.powerMeasurement.measureVccCurrent(1000, delay=1) |
if delay == 0: |
one_million_samples = 1 * 1000 * 1000 |
self.lib.powerMeasurement.measureVccqVccq2Current(samples=2 * one_million_samples, sampleRate=250, test="Idle", |
temperature="25C", hibernate=True) |
self.logger.info("Step2.2-----Exit hibernate -------") |
response = self.lun[self.lun.keys()[0]].Dme.HibernateExit() |
if ((response.Argument2 & 0xFF) != 0): |
self.addFailure("Hibern8 Exit DME command failed. error code is {} ".format(response.Argument2)) |
else: |
self.logger.info("exec time of exit hibernate is {}us".format(response.ExecTime)) |
H8_Exit_lat.append(response.ExecTime) |
def getBigLunId(self): |
lunList = self.lib.reportLuns.getAvailableNormalLuns() |
bigLun = 0 |
maxLbaBigLun = 0 |
for lunId in lunList: |
buf = self.mist.Buffer(1, 8) |
tempLba = self.lun[lunId].ReadCapacity10Sync(buf).MaxLba |
if tempLba > maxLbaBigLun: |
bigLun = lunId |
maxLbaBigLun = tempLba |
maxUnmapLbaSize = self.lib.unmapAsync._getMaxUnmapLbaSize(bigLun) |
intCapacityGB = maxLbaBigLun / self.lbaStepSize |
maxLba = intCapacityGB * self.lbaStepSize |
lbaMainStart = 0 |
LbaRange = [(startLba, endLba) for startLba, endLba in |
zip(range(lbaMainStart, maxLba - self.lbaStepSize, self.lbaStepSize), range(lbaMainStart + self.lbaStepSize - 1, maxLba, self.lbaStepSize))] |
return bigLun, maxLbaBigLun, LbaRange, maxUnmapLbaSize |
def getLatencies(self, cmdCalls, isSequential, secIdleBoundary): |
usLatencies = self.lib.performanceAdHoc.loopCmd( |
command=cmdCalls, loopSize=self.totalCmdNum, sequential=isSequential, idleSleep=secIdleBoundary, verbose=self.verbose) |
return usLatencies |
def Test(self): |
""" |
This is the main test function of the TC Script. |
""" |
self._eraseFullCardPost() |
self.logger.info("<Step1> PRECONDITION: Lun config") |
self._newTargetConfg() |
self.lunID, self.maxLba, self.LbaRange, self.maxUnmapLbaSize = self.getBigLunId() |
self.vuCacheDict = collections.OrderedDict() |
self.vuLatencyTO = self.vuCMDTO |
self.logger.info("Lun {}".format(self.lunID)) |
# ####### VARIABLES ####### # |
cmdCalls, cmdNames, isSequential = self._getVariables(self.lunID) |
self.cmdNames = cmdNames |
# ####### PRECONDITION ####### # |
self.logger.info("<Step2> Send 1000 * TUR to measure host overhead ") |
oldValues, avgTeUnRe, idleBoundary = self._preCondition(lunID=self.lunID) |
secIdleBoundary = self.lib.generic.changeUnits("seconds", self.outputUnit, idleBoundary) |
# ####### MAIN ####### # |
self.logger.info("<Step3> Send {} * {} commands to measure latency".format(self.totalCmdNum, cmdNames)) |
usLatencies = self.getLatencies(cmdCalls, isSequential, secIdleBoundary) |
latencies = self._afterLoopBeforeAveraging(usLatencies=usLatencies) |
self._saveRawData(self.cmdNames, latencies) |
# make sure to use len(latencies) in case of duplicated cmdNames |
for i in xrange(len(latencies)): |
allData = self._calAllData(self.type, cmdNames[i], latencies[i]) |
self._handleLatencies(allData, self.logFileName[:-4]) |
self.logger.debug("POSTCONDITION") |
self._postCondition(lunID=self.lunID, oldValues=oldValues) |
self._eraseFullCardPost() |
def Main(): |
test = Latency_00_Command_Base() |
exit(test.Run()) |
if __name__ == "__main__": |
Main() |
Latency.rar(免费下载)[11 次下载]
中级程序员
by: 小鬼哈哈哈 发表于：2020-02-25 16:34:30 顶(0) | 踩(0) 回复
这个是干嘛的？
回复评论