Commit 650ab98d authored by Lluís Vilanova's avatar Lluís Vilanova Committed by Stefan Hajnoczi

tracetool: Rewrite infrastructure as python modules

The tracetool script is written in shell and has hit several portability
problems due to shell quirks or external tools across host platforms.
Additionally the amount of string processing and lack of real data
structures makes it tough to implement code generator backends for
tracers that are more complex.

This patch replaces the shell version of tracetool with a Python
version.  The new tracetool design is:

  scripts/tracetool.py - top-level script
  scripts/tracetool/backend/ - tracer backends live here (simple, ust)
  scripts/tracetool/format/  - output formats live here (.c, .h)

There is common code for trace-events definition parsing so that
backends can focus on generating code rather than parsing input.

Support for all existing backends (nop, stderr, simple, ust,
and dtrace) is added back in follow-up patches.

[Commit description written by Stefan Hajnoczi]
Signed-off-by: default avatarLluís Vilanova <vilanova@ac.upc.edu>
Signed-off-by: default avatarStefan Hajnoczi <stefanha@linux.vnet.ibm.com>
parent 6e7a7f3d
......@@ -378,12 +378,12 @@ else
trace.h: trace.h-timestamp
endif
trace.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
$(call quiet-command,sh $(SRC_PATH)/scripts/tracetool --$(TRACE_BACKEND) -h < $< > $@," GEN trace.h")
$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py --format=h --backend=$(TRACE_BACKEND) < $< > $@," GEN trace.h")
@cmp -s $@ trace.h || cp $@ trace.h
trace.c: trace.c-timestamp
trace.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
$(call quiet-command,sh $(SRC_PATH)/scripts/tracetool --$(TRACE_BACKEND) -c < $< > $@," GEN trace.c")
$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py --format=c --backend=$(TRACE_BACKEND) < $< > $@," GEN trace.c")
@cmp -s $@ trace.c || cp $@ trace.c
trace.o: trace.c $(GENERATED_HEADERS)
......@@ -396,7 +396,7 @@ trace-dtrace.h: trace-dtrace.dtrace
# rule file. So we use '.dtrace' instead
trace-dtrace.dtrace: trace-dtrace.dtrace-timestamp
trace-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
$(call quiet-command,sh $(SRC_PATH)/scripts/tracetool --$(TRACE_BACKEND) -d < $< > $@," GEN trace-dtrace.dtrace")
$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py --format=d --backend=$(TRACE_BACKEND) < $< > $@," GEN trace-dtrace.dtrace")
@cmp -s $@ trace-dtrace.dtrace || cp $@ trace-dtrace.dtrace
trace-dtrace.o: trace-dtrace.dtrace $(GENERATED_HEADERS)
......
......@@ -59,12 +59,13 @@ TARGET_TYPE=system
endif
$(QEMU_PROG).stp: $(SRC_PATH)/trace-events
$(call quiet-command,sh $(SRC_PATH)/scripts/tracetool \
--$(TRACE_BACKEND) \
--binary $(bindir)/$(QEMU_PROG) \
--target-arch $(TARGET_ARCH) \
--target-type $(TARGET_TYPE) \
--stap < $(SRC_PATH)/trace-events > $(QEMU_PROG).stp," GEN $(QEMU_PROG).stp")
$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py \
--format=stap \
--backend=$(TRACE_BACKEND) \
--binary=$(bindir)/$(QEMU_PROG) \
--target-arch=$(TARGET_ARCH) \
--target-type=$(TARGET_TYPE) \
< $(SRC_PATH)/trace-events > $(QEMU_PROG).stp," GEN $(QEMU_PROG).stp")
else
stap:
endif
......
......@@ -1097,7 +1097,7 @@ echo " --disable-docs disable documentation build"
echo " --disable-vhost-net disable vhost-net acceleration support"
echo " --enable-vhost-net enable vhost-net acceleration support"
echo " --enable-trace-backend=B Set trace backend"
echo " Available backends:" $("$source_path"/scripts/tracetool --list-backends)
echo " Available backends:" $($python "$source_path"/scripts/tracetool.py --list-backends)
echo " --with-trace-file=NAME Full PATH,NAME of file to store traces"
echo " Default:trace-<pid>"
echo " --disable-spice disable spice"
......@@ -2670,7 +2670,7 @@ fi
##########################################
# check if trace backend exists
sh "$source_path/scripts/tracetool" "--$trace_backend" --check-backend > /dev/null 2> /dev/null
$python "$source_path/scripts/tracetool.py" "--backend=$trace_backend" --check-backend > /dev/null 2> /dev/null
if test "$?" -ne 0 ; then
echo
echo "Error: invalid trace backend"
......
This diff is collapsed.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Command-line wrapper for the tracetool machinery.
"""
__author__ = "Lluís Vilanova <vilanova@ac.upc.edu>"
__copyright__ = "Copyright 2012, Lluís Vilanova <vilanova@ac.upc.edu>"
__license__ = "GPL version 2 or (at your option) any later version"
__maintainer__ = "Stefan Hajnoczi"
__email__ = "stefanha@linux.vnet.ibm.com"
import sys
import getopt
from tracetool import error_write, out
import tracetool.backend
import tracetool.format
_SCRIPT = ""
def error_opt(msg = None):
if msg is not None:
error_write("Error: " + msg + "\n")
backend_descr = "\n".join([ " %-15s %s" % (n, d)
for n,d in tracetool.backend.get_list() ])
format_descr = "\n".join([ " %-15s %s" % (n, d)
for n,d in tracetool.format.get_list() ])
error_write("""\
Usage: %(script)s --format=<format> --backend=<backend> [<options>]
Backends:
%(backends)s
Formats:
%(formats)s
Options:
--help This help message.
--list-backends Print list of available backends.
--check-backend Check if the given backend is valid.
""" % {
"script" : _SCRIPT,
"backends" : backend_descr,
"formats" : format_descr,
})
if msg is None:
sys.exit(0)
else:
sys.exit(1)
def main(args):
global _SCRIPT
_SCRIPT = args[0]
long_opts = [ "backend=", "format=", "help", "list-backends", "check-backend" ]
long_opts += [ "binary=", "target-type=", "target-arch=", "probe-prefix=" ]
try:
opts, args = getopt.getopt(args[1:], "", long_opts)
except getopt.GetoptError as err:
error_opt(str(err))
check_backend = False
arg_backend = ""
arg_format = ""
for opt, arg in opts:
if opt == "--help":
error_opt()
elif opt == "--backend":
arg_backend = arg
elif opt == "--format":
arg_format = arg
elif opt == "--list-backends":
backends = tracetool.backend.get_list()
out(", ".join([ b for b,_ in backends ]))
sys.exit(0)
elif opt == "--check-backend":
check_backend = True
else:
error_opt("unhandled option: %s" % opt)
if arg_backend is None:
error_opt("backend not set")
if check_backend:
if tracetool.backend.exists(arg_backend):
sys.exit(0)
else:
sys.exit(1)
try:
tracetool.generate(sys.stdin, arg_format, arg_backend)
except tracetool.TracetoolError as e:
error_opt(str(e))
if __name__ == "__main__":
main(sys.argv)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Machinery for generating tracing-related intermediate files.
"""
__author__ = "Lluís Vilanova <vilanova@ac.upc.edu>"
__copyright__ = "Copyright 2012, Lluís Vilanova <vilanova@ac.upc.edu>"
__license__ = "GPL version 2 or (at your option) any later version"
__maintainer__ = "Stefan Hajnoczi"
__email__ = "stefanha@linux.vnet.ibm.com"
import re
import sys
import tracetool.format
import tracetool.backend
def error_write(*lines):
"""Write a set of error lines."""
sys.stderr.writelines("\n".join(lines) + "\n")
def error(*lines):
"""Write a set of error lines and exit."""
error_write(*lines)
sys.exit(1)
def out(*lines, **kwargs):
"""Write a set of output lines.
You can use kwargs as a shorthand for mapping variables when formating all
the strings in lines.
"""
lines = [ l % kwargs for l in lines ]
sys.stdout.writelines("\n".join(lines) + "\n")
class Arguments:
"""Event arguments description."""
def __init__(self, args):
"""
Parameters
----------
args :
List of (type, name) tuples.
"""
self._args = args
@staticmethod
def build(arg_str):
"""Build and Arguments instance from an argument string.
Parameters
----------
arg_str : str
String describing the event arguments.
"""
res = []
for arg in arg_str.split(","):
arg = arg.strip()
parts = arg.split()
head, sep, tail = parts[-1].rpartition("*")
parts = parts[:-1]
if tail == "void":
assert len(parts) == 0 and sep == ""
continue
arg_type = " ".join(parts + [ " ".join([head, sep]).strip() ]).strip()
res.append((arg_type, tail))
return Arguments(res)
def __iter__(self):
"""Iterate over the (type, name) pairs."""
return iter(self._args)
def __len__(self):
"""Number of arguments."""
return len(self._args)
def __str__(self):
"""String suitable for declaring function arguments."""
if len(self._args) == 0:
return "void"
else:
return ", ".join([ " ".join([t, n]) for t,n in self._args ])
def __repr__(self):
"""Evaluable string representation for this object."""
return "Arguments(\"%s\")" % str(self)
def names(self):
"""List of argument names."""
return [ name for _, name in self._args ]
def types(self):
"""List of argument types."""
return [ type_ for type_, _ in self._args ]
class Event(object):
"""Event description.
Attributes
----------
name : str
The event name.
fmt : str
The event format string.
properties : set(str)
Properties of the event.
args : Arguments
The event arguments.
"""
_CRE = re.compile("((?P<props>.*)\s+)?(?P<name>[^(\s]+)\((?P<args>[^)]*)\)\s*(?P<fmt>\".*)?")
_VALID_PROPS = set(["disable"])
def __init__(self, name, props, fmt, args):
"""
Parameters
----------
name : string
Event name.
props : list of str
Property names.
fmt : str
Event printing format.
args : Arguments
Event arguments.
"""
self.name = name
self.properties = props
self.fmt = fmt
self.args = args
unknown_props = set(self.properties) - self._VALID_PROPS
if len(unknown_props) > 0:
raise ValueError("Unknown properties: %s" % ", ".join(unknown_props))
@staticmethod
def build(line_str):
"""Build an Event instance from a string.
Parameters
----------
line_str : str
Line describing the event.
"""
m = Event._CRE.match(line_str)
assert m is not None
groups = m.groupdict('')
name = groups["name"]
props = groups["props"].split()
fmt = groups["fmt"]
args = Arguments.build(groups["args"])
return Event(name, props, fmt, args)
def __repr__(self):
"""Evaluable string representation for this object."""
return "Event('%s %s(%s) %s')" % (" ".join(self.properties),
self.name,
self.args,
self.fmt)
def _read_events(fobj):
res = []
for line in fobj:
if not line.strip():
continue
if line.lstrip().startswith('#'):
continue
res.append(Event.build(line))
return res
class TracetoolError (Exception):
"""Exception for calls to generate."""
pass
def try_import(mod_name, attr_name = None, attr_default = None):
"""Try to import a module and get an attribute from it.
Parameters
----------
mod_name : str
Module name.
attr_name : str, optional
Name of an attribute in the module.
attr_default : optional
Default value if the attribute does not exist in the module.
Returns
-------
A pair indicating whether the module could be imported and the module or
object or attribute value.
"""
try:
module = __import__(mod_name, fromlist=["__package__"])
if attr_name is None:
return True, module
return True, getattr(module, str(attr_name), attr_default)
except ImportError:
return False, None
def generate(fevents, format, backend):
"""Generate the output for the given (format, backend) pair.
Parameters
----------
fevents : file
Event description file.
format : str
Output format name.
backend : str
Output backend name.
"""
# fix strange python error (UnboundLocalError tracetool)
import tracetool
format = str(format)
if len(format) is 0:
raise TracetoolError("format not set")
mformat = format.replace("-", "_")
if not tracetool.format.exists(mformat):
raise TracetoolError("unknown format: %s" % format)
backend = str(backend)
if len(backend) is 0:
raise TracetoolError("backend not set")
mbackend = backend.replace("-", "_")
if not tracetool.backend.exists(mbackend):
raise TracetoolError("unknown backend: %s" % backend)
if not tracetool.backend.compatible(mbackend, mformat):
raise TracetoolError("backend '%s' not compatible with format '%s'" %
(backend, format))
events = _read_events(fevents)
if backend == "nop":
( e.properies.add("disable") for e in events )
tracetool.format.generate_begin(mformat, events)
tracetool.backend.generate("nop", format,
[ e
for e in events
if "disable" in e.properties ])
tracetool.backend.generate(backend, format,
[ e
for e in events
if "disable" not in e.properties ])
tracetool.format.generate_end(mformat, events)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Backend management.
Creating new backends
---------------------
A new backend named 'foo-bar' corresponds to Python module
'tracetool/backend/foo_bar.py'.
A backend module should provide a docstring, whose first non-empty line will be
considered its short description.
All backends must generate their contents through the 'tracetool.out' routine.
Backend functions
-----------------
======== =======================================================================
Function Description
======== =======================================================================
<format> Called to generate the format- and backend-specific code for each of
the specified events. If the function does not exist, the backend is
considered not compatible with the given format.
======== =======================================================================
"""
__author__ = "Lluís Vilanova <vilanova@ac.upc.edu>"
__copyright__ = "Copyright 2012, Lluís Vilanova <vilanova@ac.upc.edu>"
__license__ = "GPL version 2 or (at your option) any later version"
__maintainer__ = "Stefan Hajnoczi"
__email__ = "stefanha@linux.vnet.ibm.com"
import pkgutil
import tracetool
def get_list():
"""Get a list of (name, description) pairs."""
res = [("nop", "Tracing disabled.")]
for _, modname, _ in pkgutil.iter_modules(tracetool.backend.__path__):
module = tracetool.try_import("tracetool.backend." + modname)
# just in case; should never fail unless non-module files are put there
if not module[0]:
continue
module = module[1]
doc = module.__doc__
if doc is None:
doc = ""
doc = doc.strip().split("\n")[0]
name = modname.replace("_", "-")
res.append((name, doc))
return res
def exists(name):
"""Return whether the given backend exists."""
if len(name) == 0:
return False
if name == "nop":
return True
name = name.replace("-", "_")
return tracetool.try_import("tracetool.backend." + name)[1]
def compatible(backend, format):
"""Whether a backend is compatible with the given format."""
if not exists(backend):
raise ValueError("unknown backend: %s" % backend)
backend = backend.replace("-", "_")
format = format.replace("-", "_")
if backend == "nop":
return True
else:
func = tracetool.try_import("tracetool.backend." + backend,
format, None)[1]
return func is not None
def _empty(events):
pass
def generate(backend, format, events):
"""Generate the per-event output for the given (backend, format) pair."""
if not compatible(backend, format):
raise ValueError("backend '%s' not compatible with format '%s'" %
(backend, format))
backend = backend.replace("-", "_")
format = format.replace("-", "_")
if backend == "nop":
func = tracetool.try_import("tracetool.format." + format,
"nop", _empty)[1]
else:
func = tracetool.try_import("tracetool.backend." + backend,
format, None)[1]
func(events)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Format management.
Creating new formats
--------------------
A new format named 'foo-bar' corresponds to Python module
'tracetool/format/foo_bar.py'.
A format module should provide a docstring, whose first non-empty line will be
considered its short description.
All formats must generate their contents through the 'tracetool.out' routine.
Format functions
----------------
All the following functions are optional, and no output will be generated if
they do not exist.
======== =======================================================================
Function Description
======== =======================================================================
begin Called to generate the format-specific file header.
end Called to generate the format-specific file footer.
nop Called to generate the per-event contents when the event is disabled or
the selected backend is 'nop'.
======== =======================================================================
"""
__author__ = "Lluís Vilanova <vilanova@ac.upc.edu>"
__copyright__ = "Copyright 2012, Lluís Vilanova <vilanova@ac.upc.edu>"
__license__ = "GPL version 2 or (at your option) any later version"
__maintainer__ = "Stefan Hajnoczi"
__email__ = "stefanha@linux.vnet.ibm.com"
import pkgutil
import tracetool
def get_list():
"""Get a list of (name, description) pairs."""
res = []
for _, modname, _ in pkgutil.iter_modules(tracetool.format.__path__):
module = tracetool.try_import("tracetool.format." + modname)
# just in case; should never fail unless non-module files are put there
if not module[0]:
continue
module = module[1]
doc = module.__doc__
if doc is None:
doc = ""
doc = doc.strip().split("\n")[0]
name = modname.replace("_", "-")
res.append((name, doc))
return res
def exists(name):
"""Return whether the given format exists."""
if len(name) == 0:
return False
name = name.replace("-", "_")
return tracetool.try_import("tracetool.format." + name)[1]
def _empty(events):
pass
def generate_begin(name, events):
"""Generate the header of the format-specific file."""
if not exists(name):
raise ValueError("unknown format: %s" % name)
name = name.replace("-", "_")
func = tracetool.try_import("tracetool.format." + name,
"begin", _empty)[1]
func(events)
def generate_end(name, events):
"""Generate the footer of the format-specific file."""
if not exists(name):
raise ValueError("unknown format: %s" % name)
name = name.replace("-", "_")
func = tracetool.try_import("tracetool.format." + name,
"end", _empty)[1]
func(events)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment