Skip to content

Commit

Permalink
Merge pull request chipsalliance#115 from antmicro/mczyz/verification
Browse files Browse the repository at this point in the history
Verification: add nox and common Makefile to manage test suite
  • Loading branch information
tmichalak authored Sep 15, 2023
2 parents 774510e + 9d05f2c commit c2f37c0
Show file tree
Hide file tree
Showing 6 changed files with 257 additions and 0 deletions.
21 changes: 21 additions & 0 deletions verification/block/.flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
; Copyright (C) 2023 Antmicro
; SPDX-License-Identifier: Apache-2.0
[flake8]
ignore = E203, E266, E501, W503, F403, F401, F405
max-line-length = 100
max-complexity = 27
select = B,C,E,F,W,T4,B9
exclude =
.git,
.gitignore,
.gitmodules,
.github,
.nox,
.pytest_cache,
__pycache__,
docs/source/conf.py,
venv,
count = True
show-source = True
statistics = True

File renamed without changes.
59 changes: 59 additions & 0 deletions verification/block/common.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Copyright (c) 2023 Antmicro <www.antmicro.com>
# SPDX-License-Identifier: Apache-2.0

TOPLEVEL_LANG = verilog
SIM ?= verilator
WAVES ?= 1

# Paths
CURDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
CFGDIR := $(abspath $(CURDIR)/snapshots/default)
CONFIG := $(abspath $(CURDIR)/../../configs)

# Common sources
COMMON_SOURCES = $(CFGDIR)/common_defines.vh
COMMON_SOURCES += $(CFGDIR)/el2_pdef.vh
COMMON_SOURCES += $(SRCDIR)/include/el2_def.sv
COMMON_SOURCES += $(SRCDIR)/lib/beh_lib.sv

VERILOG_SOURCES := $(COMMON_SOURCES) $(VERILOG_SOURCES)

# Coverage reporting
COVERAGE_TYPE ?= ""
ifeq ("$(COVERAGE_TYPE)", "all")
VERILATOR_COVERAGE = --coverage
else ifeq ("$(COVERAGE_TYPE)", "branch")
VERILATOR_COVERAGE = --coverage-line
else ifeq ("$(COVERAGE_TYPE)", "toggle")
VERILATOR_COVERAGE = --coverage-toggle
else ifeq ("$(COVERAGE_TYPE)", "functional")
VERILATOR_COVERAGE = --coverage-user
else
VERILATOR_COVERAGE = ""
endif

# Enable processing of #delay statements
ifeq ($(SIM), verilator)
COMPILE_ARGS += --timing
COMPILE_ARGS += -Wall -Wno-fatal

EXTRA_ARGS += --trace --trace-structs
EXTRA_ARGS += $(VERILATOR_COVERAGE)
endif

COCOTB_HDL_TIMEUNIT = 1ns
COCOTB_HDL_TIMEPRECISION = 10ps

EXTRA_ARGS += -I$(CFGDIR)

# Build directory
ifneq ($(COVERAGE_TYPE),)
SIM_BUILD := sim-build-$(COVERAGE_TYPE)
endif

include $(shell cocotb-config --makefiles)/Makefile.sim

# Rules for generating VeeR config
$(CFGDIR)/common_defines.vh:
cd $(CURDIR) && $(CONFIG)/veer.config -fpga_optimize=0

144 changes: 144 additions & 0 deletions verification/block/noxfile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
# Copyright (C) 2023 Antmicro
# SPDX-License-Identifier: Apache-2.0

import logging
import os
from xml.etree import ElementTree as ET

import nox

# nox quirk: in status.json, return code for failure is 0
# https://github.com/wntrblm/nox/blob/main/nox/sessions.py#L128C11-L128C11
nox.options.report = "status.json"
nox.options.reuse_existing_virtualenvs = True

# Test configuration
blockPath = "."
pipRequirementsPath = "requirements.txt"

# Coverage types to collect
coverageTypes = [
"all",
"branch",
"toggle",
]


def isSimFailure(
resultsFile="results.xml", testsuites_name="results", verbose=False, suppress_rc=False
):
"""
Extract failure code from cocotb results.xml file
Based on https://github.com/cocotb/cocotb/blob/master/bin/combine_results.py
"""
rc = 0

# Logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logHandler = logging.FileHandler(filename="parseResultsXML.log", mode="w", encoding="utf-8")
logFormatter = logging.Formatter()
logHandler.setFormatter(logFormatter)
logger.addHandler(logHandler)
logHandler.setLevel(logging.INFO)
if verbose:
logHandler.setLevel(logging.DEBUG)

# Main
result = ET.Element("testsuites", name=testsuites_name)
logging.debug(f"Reading file {resultsFile}")
tree = ET.parse(resultsFile)

for ts in tree.iter("testsuite"):
ts_name = ts.get("name")
ts_package = ts.get("package")
logging.debug(f"Testsuite name : {ts_name}, package : {ts_package}")
use_element = None
for existing in result:
if existing.get("name") == ts.get("name") and existing.get("package") == ts.get(
"package"
):
use_element = existing
break
if use_element is None:
result.append(ts)
else:
use_element.extend(list(ts))

if verbose:
ET.dump(result)

for testsuite in result.iter("testsuite"):
for testcase in testsuite.iter("testcase"):
for failure in testcase.iter("failure"):
rc = 1
logging.info(
"Failure in testsuite: '{}' classname: '{}' testcase: '{}' with parameters '{}'".format(
testsuite.get("name"),
testcase.get("classname"),
testcase.get("name"),
testsuite.get("package"),
)
)

if suppress_rc:
rc = 0
logging.shutdown()
return rc


def verify_block(session, blockName, testName, coverage=""):
session.install("-r", pipRequirementsPath)
testPath = os.path.join(blockPath, blockName)
testNameXML = os.path.join(testName + ".xml")
testNameXMLPath = os.path.join(testPath, testNameXML)
testNameLog = os.path.join(testName + ".log")
testNameLogPath = os.path.join(testPath, testNameLog)
with open(testNameLogPath, "w") as testLog:
session.run(
"make",
"-C",
testPath,
"all",
"COVERAGE_TYPE=" + coverage,
"MODULE=" + testName,
"COCOTB_RESULTS_FILE=" + testNameXML,
stdout=testLog,
stderr=testLog,
)
# Prevent coverage.dat and test log from being overwritten
if coverage != "":
coveragePath = testPath
coverageName = "coverage.dat"
coverageNamePath = os.path.join(coveragePath, coverageName)
newCoverageName = "coverage_" + testName + "_" + coverage + ".dat"
newCoverageNamePath = os.path.join(coveragePath, newCoverageName)
os.rename(coverageNamePath, newCoverageNamePath)
newTestNameLog = testName + "_" + coverage + ".log"
newTestNameLogPath = os.path.join(testPath, newTestNameLog)
os.rename(testNameLogPath, newTestNameLogPath)
# Add check from results.xml to notify nox that test failed
isTBFailure = isSimFailure(resultsFile=testNameXMLPath)
if isTBFailure:
raise Exception("SimFailure: cocotb failed. See test logs for more information.")


@nox.session()
def isort(session: nox.Session) -> None:
"""Options are defined in pyproject.toml file"""
session.install("isort")
session.run("isort", ".")


@nox.session()
def flake8(session: nox.Session) -> None:
"""Options are defined in .flake8 file."""
session.install("flake8")
session.run("flake8", ".")


@nox.session()
def black(session: nox.Session) -> None:
"""Options are defined in pyproject.toml file"""
session.install("black")
session.run("black", ".")
24 changes: 24 additions & 0 deletions verification/block/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Copyright (C) 2023 Antmicro
# SPDX-License-Identifier: Apache-2.0

[tool.black]
line-length = 100
exclude = '''
(
/(
| \.git
| \.gitignore
| \.gitmodules
| \.github
| \.nox
| \.pytest_cache
| __pycache__
| venv
)/
| docs/source/conf.py
)
'''

[tool.isort]
profile = "black"
multi_line_output = 3
9 changes: 9 additions & 0 deletions verification/block/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
cocotb==1.8.0
cocotb-bus==0.2.1
cocotb-coverage==1.1.0
cocotb-test==0.2.4
pytest==7.4.1
pytest-html==3.2.0
pytest-timeout==2.1.0
pytest-md==0.2.0
pyuvm==2.9.1

0 comments on commit c2f37c0

Please sign in to comment.