# ----------------------------------------------------------------------------- # Title : Memory Mapper base class script # Project : Generic FW tools # ----------------------------------------------------------------------------- # File : memMapperBase.py # Author : Simon Southwell # Company : Global Inkjet Systems # Created : 2017-12-04 # Last update: 2017-12-11 # Platform : # Standard : Python 3.x # ----------------------------------------------------------------------------- # Description: # Definition of an abstract base class for reading memory mapped register # definitions from JSON file and providing a python dictionary model to a # virtual function to be defined in a child class that inherits this class. # ----------------------------------------------------------------------------- # Copyright (c) 2017 Global Inkjet Systems # ----------------------------------------------------------------------------- import sys import os import re import json import collections # --------------------------------------------------------- # class to define string literals for valid attributes # --------------------------------------------------------- class attr : IP_NAME = 'ip_name' BUS = 'bus' ADDR_WIDTH = 'addr_width' ADDR_HI_BIT = 'addr_hi_bit' ADDR_LO_BIT = 'addr_lo_bit' REGISTERS = 'registers' ADDRESS = 'address' ADDR_RANGE = 'addr_range' TYPE = 'type' RESET = 'reset' WIDTH = 'width' DESCRIPTION = 'description' FIELDS = 'fields' BIT_LEN = 'bit_len' SUB_BLOCKS = 'sub_blocks' INSTANCE = 'instance' OFFSET = 'offset' PROCESS = 'process' PROCESS_REGS = 'process_regs' # --------------------------------------------------------- # class to define string literals for valid registers types # --------------------------------------------------------- class regtype : WRITE = 'w' WRITE_PULSE = 'wp' WRITE_CLR0 = 'w0' READ = 'r' READ_PULSE = 'rp' CONSTANT = 'c' RESERVED = 'rsv' READ_WRITE_PULSE = 'rwp' typeList = [WRITE, WRITE_PULSE, WRITE_CLR0, READ, READ_PULSE, CONSTANT, RESERVED, READ_WRITE_PULSE] # --------------------------------------------------------- # Abstract base class to locate JSON descriptions of # memory mapped registers for a project, read them in # and output formatted information based on the model # produced. The output method is virtual and not defined # in this class, but is defined in a child class with # specifics defined there, such as VHDL, HAL or text # output. # --------------------------------------------------------- class MemMapperBase : # Some 'constants' defining which register/field types are read or write # used in validation of model writeTypes = [regtype.WRITE, regtype.WRITE_CLR0, regtype.WRITE_PULSE, regtype.READ_WRITE_PULSE] readTypes = [regtype.READ, regtype.READ_PULSE, regtype.CONSTANT] neutralTypes = [regtype.RESERVED] noReadTypes = [regtype.WRITE_CLR0, regtype.RESERVED] wrPulseTypes = [regtype.WRITE_PULSE, regtype.READ_WRITE_PULSE] rdPulseTypes = [regtype.READ_PULSE, regtype.READ_WRITE_PULSE] pulseTypes = wrPulseTypes + rdPulseTypes # The valid types are all the possible types validTypes = writeTypes + readTypes + neutralTypes # --------------------------------------------------------- # Constructor : initialise class member variables # --------------------------------------------------------- def __init__(self) : self.docDirs = [] self.jsonPaths = {} self.busList = [] self.subBlockOffsetsDict = {} # --------------------------------------------------------- # Virtual function (public method) # --------------------------------------------------------- # def genOutput(self, ip): # print('***ERROR: Called base class directly. ' + # 'Please use one of the derived child classes with locally defined genOutput() method') # sys.exit(1) # --------------------------------------------------------- # Create a package name tag # --------------------------------------------------------- @staticmethod def createPkgTag(bus, regName, isAddr = True) : tagStr = bus.upper() + '_' + regName.upper() if isAddr : tagStr += '_ADDR' return tagStr # --------------------------------------------------------- # Convert a value to a std_logic or std_logic_vector # literal # --------------------------------------------------------- @staticmethod def valToStdLogic (val, width) : if width == 1: valStr = '\'' + str(val) + '\'' else: if val == 0: valStr = '(others => \'0\')' else: nibbles = (width + 3) >> 2 fmtStr = '0' + str(nibbles) + 'X' hexStr = '16#' + format(val, fmtStr) + '#' valStr = 'std_logic_vector(to_unsigned(' + hexStr + ', ' + str(width) + '))' return valStr # --------------------------------------------------------- # Method to generate a formatted VHDL hex vectoor literal # from a value, with the right number of padding nibbles # --------------------------------------------------------- @staticmethod def valToVhdlHexVector(val, width): nibbles = (width + 3) >> 2 fmtStr = '0' + str(nibbles) + 'X' return 'x"' + format(val, fmtStr) + '"' # --------------------------------------------------------- # Underscore to camel case # --------------------------------------------------------- @staticmethod def _underScore2CamelCase(usStr) : lowCaseName = usStr.lower() nameSpiltList = lowCaseName.split('_') nameCamelList = [x.capitalize() for x in nameSpiltList] return ''.join(nameCamelList) # --------------------------------------------------------- # Underscore to camel case # --------------------------------------------------------- @staticmethod def _underScore2LowCamelCase(usStr): lowCaseName = usStr.lower() nameSpiltList = lowCaseName.split('_') nameCamelList = [x.capitalize() for x in nameSpiltList[1:]] return ''.join([nameSpiltList[0]] + nameCamelList) # --------------------------------------------------------- # Find all the 'doc' directories which is where the JSON # files will be found. (private method) # --------------------------------------------------------- @staticmethod def __getDocDirs(rootDir) : return [x[0] for x in os.walk(rootDir) if x[0].endswith('doc')] # --------------------------------------------------------- # Find the directory for a particular JSON file. # (private method) # --------------------------------------------------------- @staticmethod def __getJsonPath(blockName, docDirs) : fileName = '.'.join([blockName, 'json']) found = 0 totalPath = '' for idx in docDirs: fileList = os.listdir(idx) if fileName in fileList: totalPath = os.sep.join([idx, fileName]) found = 1 if found == 0: print('Could not find: ' + fileName) return os.path.abspath(totalPath) # --------------------------------------------------------- # Go through the IP list and find the directories for # the JSON files. (private method) # --------------------------------------------------------- def __getJsonPaths(self, jsonPaths, ipArray): for directory in self.docDirs: fileList = os.listdir(directory) for ip in ipArray: fileName = ip+'.json' if fileName in fileList: jsonPaths[ip] = os.sep.join([directory, fileName]) # --------------------------------------------------------- # Extract the dictionary of information from the JSON file # (private method) # --------------------------------------------------------- @staticmethod def __getJsonDict(blockName, jsonPaths) : jsonFileName = jsonPaths[blockName] jsonFile = open(jsonFileName) jsonStr = jsonFile.read() # Load in the JSON data, and use ordered dictionaries from collections package # in place of built-in dictionaries, to ensure registers and fields are not # presented in random order when iterating through entries try : jsonData = json.loads(jsonStr, object_pairs_hook=collections.OrderedDict) except ValueError as err: sys.stdout.write('Error in JSON file while parsing ' + blockName + ': ') print(err) sys.exit(1) return jsonData # --------------------------------------------------------- # Find all the 'build' directories which is where will find # QSYS files. (private method) # --------------------------------------------------------- @staticmethod def __getBuildDirs(rootDir, buildDirName='build') : return [x[0] for x in os.walk(rootDir) if x[0].endswith(buildDirName)] # --------------------------------------------------------- # Find which directory a particular QSYS file live in # (private method) # --------------------------------------------------------- @staticmethod def __getQsysPath(blockName, buildDirs) : fileName = blockName + '.qsys' # Create a Qsys filename from the block name found = 0 totalPath = '' for idx in buildDirs: fileList = os.listdir(idx) # Search amongst all the directories... if fileName in fileList: # ...that contain Qsys files totalPath = os.sep.join([idx, fileName]) found = 1 if found == 0: print('Could not find: ' + fileName) return totalPath # --------------------------------------------------------- # Search a QSYS file for references to 'baseAddress' and # extract associated IP and bus information. # (private method) # --------------------------------------------------------- @staticmethod def __getQsysInsts(topBlock, qsysPaths, busArray, qsysSuffix='_system') : lineArray = ['a', 'b', 'c', 'd'] instArray = [] readFileName = qsysPaths[topBlock + qsysSuffix] readFile = open(readFileName, 'r') for line in readFile: if ('end=' in lineArray[2]) and ('baseAddress' in line) : for bus in busArray: testStr = '.'+bus+'\"' if testStr in lineArray[2]: lineList = re.split(r'[".]', lineArray[2]) block = lineList[1] wordList = block.split('_') ip = '_'.join(wordList[:-1]) instName = wordList[-1] numList = re.split(r'["x]', line) base = numList[-2] tempArray = [bus, ip, instName, base] instArray.append(tempArray) lineArray = lineArray[1:] lineArray.append(line) return instArray # --------------------------------------------------------- # Store the IP blocks into an array. (private method) # --------------------------------------------------------- @staticmethod def __processQsysInsts(qsysInsts) : ipArray = [] for idx in range(len(qsysInsts)): if qsysInsts[idx][1] not in ipArray : ipArray.append(qsysInsts[idx][1]) return ipArray # --------------------------------------------------------- # Process all the instances (private method) # --------------------------------------------------------- def __processJsonInsts(self, ipArray, opDir='', opSuffix='') : for ip in ipArray: self.genOutput(ip, opDir, opSuffix) # Virtual function, not defined in this base class # --------------------------------------------------------- # Process all the sub-blocks and add to ipArray the entries # not already present # --------------------------------------------------------- def __processSubBlocks(self, ipArray, enableTag='') : subBlockArray = [] idx = 0 newBlocks = True # Loop, processing sub-blocks through the hierarchy, as new child modules # added while newBlocks: # Process the list of IP blocks from the last point processed # (i.e. process any added blocks) for ip in ipArray[idx:] : # Fetch the JSON path and the dictionary with the parsed data jsonPath, jsonDict = self._getModel(ip) for jsonBusDict in jsonDict: if attr.SUB_BLOCKS in jsonBusDict: subBlocks = jsonBusDict[attr.SUB_BLOCKS] for ipNameEntry in subBlocks: # Any sub-block named 'local' is not a separate block to be processed # (as it can't have child blocks, and won't have a separate JSON file) if ipNameEntry == 'local' : continue sbDict = subBlocks[ipNameEntry] processStr = sbDict[attr.PROCESS] if processStr == 'enabled' or enableTag in processStr: ipName = ipNameEntry.split(':')[0] if ipName not in subBlockArray: subBlockArray.append(ipName) if ipNameEntry in self.subBlockOffsetsDict: self.subBlockOffsetsDict[ipNameEntry].append( (sbDict['offset'], jsonBusDict[attr.BUS], ip)) else: self.subBlockOffsetsDict[ipNameEntry] = [ (sbDict['offset'], jsonBusDict[attr.BUS], ip)] # Move the index to the first of any newly added blocks idx = len(ipArray) # Add the new sub-blocks to the ipArray list ipArray += subBlockArray # Flag whether there were new blocks added or not newBlocks = len(subBlockArray) != 0 # Clear the subBlockArray subBlockArray = [] # Re-calculate the JSON paths so to include any added sub-blocks self.__getJsonPaths(self.jsonPaths, ipArray) return # --------------------------------------------------------- # Get the model data (protected method) # --------------------------------------------------------- def _getModel(self, blockName) : # Fetch the JSON path and the dictionary with the parsed data jsonPath = self.__getJsonPath(blockName, self.docDirs) jsonDict = self.__getJsonDict(blockName, self.jsonPaths) # If the validation has errors, then exit if self.__validateModel(jsonDict, blockName) : sys.exit(1) return jsonPath, jsonDict # --------------------------------------------------------- # Validate the IP model for format rules # --------------------------------------------------------- @staticmethod def __validateModel(jsonDict, ip) : errorsFound = False # Iterate through each bus dictionary for jsonBusDict in jsonDict : # Get the bus name from the dictionary bus = jsonBusDict[attr.BUS] if attr.REGISTERS not in jsonBusDict : # If no registers, must have a sub-block list if attr.SUB_BLOCKS not in jsonBusDict : errorsFound = True else : regDict = jsonBusDict[attr.REGISTERS] regKeys = regDict.keys() seenRegNames = [] seenAddresses = [] for regName in regKeys : # All registers entries must have an address if attr.ADDRESS not in regDict[regName]: print('***Error: no address found for register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True else : # No repeated addresses addrVal = int(regDict[regName][attr.ADDRESS], 0) if addrVal in seenAddresses: print('***Error: seen repeated register address ' + regDict[regName][attr.ADDRESS] + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True else: seenAddresses.append(addrVal) # All registers entries must have a description if attr.DESCRIPTION not in regDict[regName]: print('***Error: no description found for register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # All registers entries must have a width and it must be in the range 1 to 32 if attr.WIDTH not in regDict[regName]: print('***Error: no width attribute found for register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True else : widthVal = int(regDict[regName][attr.WIDTH], 0) if widthVal < 1 or widthVal > 32 : print('***Error: width value (' + regDict[regName][attr.WIDTH] + ') out of range for register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) # No repeated register names in this block if regName in seenRegNames : print('***Error: seen repeated register name ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True else : seenRegNames.append(regName) # If there are sub-fields, process them here if attr.FIELDS in regDict[regName] : # If a fields attribute present, register must not have a type if attr.TYPE in regDict[regName] : print('***Error: register has type definition when fields attribute present ' + ' for ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # If a fields attribute present, register must not have a reset attribute if attr.RESET in regDict[regName]: print('***Error: register has reset definition when fields attribute present ' + ' for ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # Extract the fields dictionary from the register fields = regDict[regName][attr.FIELDS] # There must be more than one field, if fields defined if not len(fields) > 1 : print('***Error: less than two fields in fields section defined for ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # Initialise some state to be sticky or accumulated through the fieldName loop seenWrType = False seenRdType = False bitCount = 0 # Loop through the fields for fieldName in fields : # All fields must have a description if attr.DESCRIPTION not in fields[fieldName] : print('***Error: no description attribute found for ' + fieldName + ' of register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # All fields must have a type if attr.TYPE not in fields[fieldName] : print('***Error: no type attribute found for ' + fieldName + ' of register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True else : # The type must be a valid value if fields[fieldName][attr.TYPE] not in MemMapperBase.validTypes : print('***Error: bad type attribute (' + fields[fieldName][attr.TYPE] + ') found for ' + fieldName + ' of register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # All fields must have a bit_len if attr.BIT_LEN not in fields[fieldName] : print('***Error: no bit_len attribute found for ' + fieldName + ' of register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # All fields must have a reset except rsv if attr.TYPE in fields[fieldName]: # Reset present for non-reserved fields if fields[fieldName]['type'] != regtype.RESERVED : if attr.RESET not in fields[fieldName] : print('***Error: no reset attribute found for ' + fieldName + ' of register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # If rsv type, then no reset should be present else : if attr.RESET in fields[fieldName] : print('***Error: reset attribute found for field ' + fieldName + ' of register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True if attr.BIT_LEN in fields[fieldName]: bitCount += int(fields[fieldName][attr.BIT_LEN], 0) if attr.TYPE in fields[fieldName] : fieldType = fields[fieldName][attr.TYPE] if fieldType in MemMapperBase.writeTypes : seenWrType = True if fieldType in MemMapperBase.readTypes : seenRdType = False # No mix of read and write types in fields if seenWrType and seenRdType : print('***Error: read and write types mixed in fields for register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True if attr.WIDTH in regDict[regName] : if bitCount != int(regDict[regName][attr.WIDTH], 0) : print('***Error: field bits do not add up to register\'s width for register ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # No field attribute for register... else : # If no fields attribute present, register must have a type if attr.TYPE not in regDict[regName]: print('***Error: register has no type definition when fields attribute not present ' + ' for ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True # If no fields attribute present, register must have a reset attribute if attr.RESET not in regDict[regName]: print('***Error: register has no reset definition when fields attribute not present ' + ' for ' + regName + ' when parsing ' + ip + ' on bus ' + bus) errorsFound = True return errorsFound # --------------------------------------------------------- # Parse command line for arguments common to all # derivatives. Parser object must be created externally # and passed in as an argument. # --------------------------------------------------------- @staticmethod def processCmdLine(parser) : parser.add_argument('-t', '--top_block', dest='topBlock', default='top', action='store', help='Top block name') parser.add_argument('-r', '--repo_dir', dest='repoDir', default='', action='store', help='Specify repository dir for processing') parser.add_argument('-j', '--json_file', dest='jsonFile', default='', action='store', help='Specify a specific JSON file for processing') parser.add_argument('-b', '--bus_list', dest='busList', default=['csr', 'trim'], nargs='+', help='Bus list') parser.add_argument('-o', '--output_dir', dest='opDir', default='', action='store', help='Specify output directory') parser.add_argument('-d', '--build_dir', dest='buildDir', default='build', action='store', help='Specify build directory name (default \'build\')') parser.add_argument('-s', '--file_suffix', dest='opFileSuffix', default='', action='store', help='Output file name suffix (default none)') parser.add_argument('-q', '--qsys_suffix', dest='qsysSuffix', default='_system', action='store', help='QSYS file name suffix (default \'_system\'') # --------------------------------------------------------- # run # --------------------------------------------------------- def run(self, topBlock, busNames, repoDir='', opDir='', opSuffix='', enableTag='all', jsonFile='', buildDirName='build', qsysSuffix='_system') : print('MemMapperBase') qsysPaths = {} self.busList = busNames # If no JSON specific file path specified, the process a whole repository if jsonFile == '' : # If no repository directory passed into the function, use the # location of the script, else use the argument supplied if repoDir == '' : startDir = os.path.dirname(sys.argv[0]) else : startDir = repoDir rootDir = startDir # Find the root of the repository self.docDirs = self.__getDocDirs(rootDir) # Find all the places where JSON files might be. buildDirs = self.__getBuildDirs(rootDir, buildDirName) # Find all the places where QSYS files might be. self.jsonPaths[topBlock] = self.__getJsonPath(topBlock, self.docDirs) # Find where the top level JSON file is and place this into the JSON dictionary topDict = self.__getJsonDict(topBlock, self.jsonPaths) # Get the dictionary entry for the top block topQsys = topDict[0]['qsys'] # Extract the top level Qsys system qsysPaths[topQsys] = self.__getQsysPath(topQsys, buildDirs) # Get the path of this Qsys system and place this into the Qsys dictionary qsysInsts = self.__getQsysInsts(topBlock, qsysPaths, busNames, qsysSuffix) # Find all the sub-block information in the Qsys system ipArray = self.__processQsysInsts(qsysInsts) # Put these IP blocks into an array self.__getJsonPaths(self.jsonPaths, ipArray) # Find all the related JSON files for the IP from the QSYS self.__processSubBlocks(ipArray, enableTag) # Process any sub-block JSON files, and add IP to array # If a specific JSON file path given, just process this else : # Get the name of the JSON file, and extract the IP name from it jfname = os.path.basename(jsonFile) ipname = jfname.split('.')[0] # Add the ip name to the list of IP to process (single entry list) ipArray = [ipname] # Add to the JSON path dictionary an entry for this IP's doc directory path self.jsonPaths[ipname] = os.path.abspath(jsonFile) # Add to the directory also to the docsDir list self.docDirs = [os.path.dirname(jsonFile)] # Generate files from this JSON information self.__processJsonInsts(ipArray, opDir, opSuffix)