Commit b95e05fb authored by Tomasz Wlostowski's avatar Tomasz Wlostowski

Merge remote-tracking branch 'fdsw/develop' into tom-spec-convention

parents 44b4ee21 54eafb43
[submodule "zio"]
path = zio
url = git://
Change Log
[3.0.0] - 2019-10-17
Functionalists are mainly the same as before, but the architecture has changed
and some loading mechanism as well, for this reason we decided to increase
the major version.
This diff is collapsed.
# include for buildsystem's defines
# use absolute path for REPO_PARENT
REPO_PARENT ?= $(shell /bin/pwd)/..
-include $(REPO_PARENT)/
all: kernel lib tools
# The user can override, using environment variables, all these three:
ZIO ?= zio
# ZIO_ABS has to be absolut path, due to beeing
# passed to the Kbuild
ZIO_ABS ?= $(abspath $(ZIO) )
export ZIO_ABS
ZIO_VERSION = $(shell cd $(ZIO_ABS); git describe --always --dirty --long --tags)
DIRS = $(ZIO_ABS) kernel lib tools
kernel: $(ZIO_ABS)
lib: $(ZIO_ABS)
tools: lib
DESTDIR ?= /usr/local
.PHONY: all clean modules install modules_install $(DIRS)
.PHONY: gitmodules prereq_install prereq_install_warn
install modules_install: prereq_install_warn
all clean modules install modules_install: $(DIRS)
clean: TARGET = clean
modules: TARGET = modules
install: TARGET = install
modules_install: TARGET = modules_install
$(MAKE) -C $@ $(TARGET)
@test -f .prereq_installed || \
echo -e "\n\n\tWARNING: Consider \"make prereq_install\"\n"
for d in $(SUBMOD); do $(MAKE) -C $$d modules_install || exit 1; done
touch .prereq_installed
$(ZIO_ABS): zio-init_repo
# init submodule if missing
@test -d $(ZIO_ABS)/doc || ( echo "Checking out submodule $(ZIO_ABS)" && git submodule update --init $(ZIO_ABS) )
include scripts/
Please check doc/
In ./doc/, is the source file, and you can "make" to get
pdf and other formats provided you have the proper tools installed
(mainly texinfo and tex).
# Minimal makefile for Sphinx documentation
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXBUILD ?= sphinx-build
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
$(MAKE) -C drawings
$(MAKE) doxygen TARGET=$@
GIT_VERSION = $(shell git describe --dirty --long --tags)
GIT_VERSION = $(shell git describe --dirty --long --tags)
ifeq ($(TARGET),clean)
@rm -rf doxygen-fd-output .doxystamp
$(MAKE) .doxystamp
# List of Doxygen folders to consider
DOXINPUT := ../../lib
DOXINPUT += ../../kernel/fine-delay.h
DOXEXCL := ../../lib/PyFmcFineDelay
# List of actual Doxygen source files
DOXSRC = $(shell find $(DOXINPUT) -type f -name '*.[chS]')
.doxystamp: $(DOXSRC)
@touch .doxystamp
# Configuration file for the Sphinx documentation builder.
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = 'Fmc Fine-Delay Software'
copyright = '2020, Alessandro Rubini'
author = 'Alessandro Rubini <>, Tomasz Wlostowski <>'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The master toctree document.
master_doc = 'index'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
breathe_projects = {
"fine-delay": "doxygen-fd-output/xml/",
breathe_default_project = "fine-delay"
This diff is collapsed.
# SPDX-License-Identifier: CC0-1.0
# SPDX-FileCopyrightText: 2020 CERN
PROJECT_NAME = "Fmc Fine Delay"
PROJECT_BRIEF = "Fmc Fine Delay"
OUTPUT_DIRECTORY = doxygen-fd-output
\ No newline at end of file
EPSIMAGES := $(shell ls *eps)
all: images
@rm $(EPSIMAGES:.eps=.png)
images: $(EPSIMAGES:.eps=.png)
%.png: %.eps
@convert $< $@
.PHONY: all clean images
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Figure 1: Modes of operation of FmcDelay
This diff is collapsed.
Figure 2: Definition of output timings
Driver Features
This driver is based on `zio`_ and it uses `fmc`_. It supports initial
setup of the board, setting and reading time, run-time continuous
calibration, input timestamping, output pulse generation and readback of
output settings from the hardware. It supports user-defined offsets,
so our users can tell the driver about channel-specific delays (for
example, to account for wiring) and ignore the issue in application code.
For each feature offered the driver (and documentation) the driver
tries to offer the following items; sometimes however one of them is missing
for a specific driver functionality, if we don't consider it important enough.
* A description of how the features works at low level;
* A low-level user-space program to test the actual mechanism;
* A C-language API to access the feature with data structures;
* An example program based on that API.
.. _drv_param:
Module Parameters
The driver accepts a few load-time parameters for configuration. You
can pass them to *insmod* amd *modprobe* directly, or write them
in ``/etc/modules.conf`` or the proper file in ``/etc/modutils/``.
The following parameters are used:
The parameter defaults to 0. If set, it enables more diagnostic
messages during probe (you may find it is not used, but it is
left in to be useful during further development, and avoid
compile-time changes like use of ``DEBUG``).
The period of the internal timer, if not zero.
The timer is used to poll for input events instead of enabling
the interrupt. The default interval is 0, which means to
use interrupt support. You may want to use the timer while
porting to a different carrier, before sorting out IRQ issues.
The period, in seconds, of temperature measurement to re-calibrate
the output delays. Defaults to 30. If set to zero, the
re-calibration timer is not activated.
The module also supports some more parameters that are
calibration-specific. They are described in the :ref:`Calibration<dev_cal>`
.. _zio:
.. _fmc:
Welcome to Fmc Fine-Delay Software's documentation!
.. toctree::
:maxdepth: 2
:caption: Contents:
#! /usr/bin/sed -f
# allow "%" as a comment char, but only at the beginning of the line
s/^%/@c /
#preserve blanks and braces in @example blocks
/^@example/,/^@end example/ s/{/@{/g
/^@example/,/^@end example/ s/}/@}/g
/^@example/,/^@end example/ p
/^@example/,/^@end example/ d
/^@smallexample/,/^@end smallexample/ s/{/@{/g
/^@smallexample/,/^@end smallexample/ s/}/@}/g
/^@smallexample/,/^@end smallexample/ p
/^@smallexample/,/^@end smallexample/ d
# remove leading blanks
s/^[ ]*//
This driver depends on two other modules (four
packages), as well as the Linux kernel. Also, it
must talk to a specific FPGA binary file running in the device.
Gateware Dependencies
While previous versions of this package included a gateware binary
in the ``binaries/`` subdirectory, in Jan 2014 we decided not to do
that any more. Release versions of this package are expected to
point to ``current`` gateware images for different carriers.
Clearly the driver is expected to work on any *fmc* carrier,
even those ignored to us, and we can't provide all binaries.
The up-to-date gateware binaries for the SVEC and SPEC carriers will be
always available in the *Releases* section of the Fine Delay project:
Note that the release gateware contains a stable version of
the White Rabbit PTP Core firmware. This firmware may be reloaded dynamically
at any time using carrier-specific tools.
Gateware Installation
By default, the driver looks for a gateware file named
``/lib/firmware/fmc/[carrier]-fine-delay.bin``, where ``[carrier]`` is the
carrier's name (lowercase - currently ``svec`` or ``spec``).
To install the gateware download the bitstreams from the Release page (or build
your own, as you wish) and put them in ``/lib/firmware/fmc``. You may have
to strip the version/date attached to the file names or create symlinks.
Follow your carrier instructions to load the gateware on the FPGA.
Software Dependencies
The kernel versions I am using during development is 3.10. Everything
used here is known to build with all versions from 2.6.35 to 3.12.
The driver, then, is based on the `zio`_ framework, available from
The FMC mezzanine is supported by means of the `fmc`_ software project.
Both packages (`zio`_ and `fmc`_) need to be downloaded and compiled. We do not
provide submodules because their version may conflict with other projects.
It is duty of the final user to guarantee a consistent installation.
Software Installation
To install this software package, you need to tell it where your
kernel sources live, so the package can pick the right header files.
You need to set only one environment variable:
The top-level directory of the Linux kernel you are compiling
against. If not set, the default may work if you compile in the same
host where you expect to run the driver.
Most likely, this is all you need to set. After this, you can
sudo make install LINUX=$LINUX
After installation, your carrier driver should load automatically
(for example, the PCI bus will load ``spec-fmc-carrier.ko``), but
``fmc-fine-delay.ko`` must be loaded manually, because support for automatic
loading is not yet in place. The suggested command is one or the other of
the following two:::
modprobe fmc-fine-delay [<parameter> ...] # after make install
insmod kernel/fmc-fine-delay.ko [<parameter> ...] # if not installed
Available module parameters are described in :ref:`Module Parameters<drv_param>`.
Unless you customized or want to customize one of the three
related packages, you can skip the rest of this section.
In order to compile *fine-delay* against a specific repository you can use
the following environment variables:
The top-level directory of the repository checkout of each
The top-level directory of the repository checkout of each
Headers and dependencies for the respective package are taken from the chosen
.. _zio:
.. _fmc:
This is the user manual for the *fmc-delay-1ns-4cha* board developed on Please note that the ohwr hardware project is
misnamed as *fmc-delay-1ns-8cha*; even if the board has 4
channels; the references to *8ch* below are thus correct, even if
the may seem wrong.
Repositories and Releases
The code and documentation is distributed in the following places:
This place hosts the pdf documentation for some official
release, but we prefer to use the *files* tab, below.
Read-only repositories for the software and documentation.
Read-write repositories, for those authorized.
.. note::
If you got this from the repository (as opposed to a named
*tar.gz* or *pdf* file) it may happen that you are looking at a later
commit than the release this manual claims to document.
It is a fact of life that developers forget
to re-read and fix documentation while updating the code. In that case,
please run ``git describe HEAD`` to ensure where you are.
Hardware Description
The *FMC Delay 1ns-4cha* is an FPGA Mezzanine Card (FMC - VITA 57 standard),
whose main purpose is to produce pulses delayed by a user-programmed value with
respect to the input trigger pulse. The card can also work as a Time to
Digital converter (TDC) or as a programmable pulse generator triggering at a
given TAI time.
For the sake of clarity of this document, the card's name will be further
abbreviated as *fine-delay*.
Requirements and Supported Platforms
*fine-delay* can work with any VITA 57-compliant FMC carrier, provided that
the carrier's FPGA has enough logic resources. The current software/gateware
release officially supports the following carrier and mezzanine combinations:
* CERN's SPEC (Simple PCI-Express Carrier) with one *fine-delay* mezzanine.
* CERN's SVEC (Simple VME64x Carrier) with one or two *fine-delay* mezzanines.
Note that if only one *fine-delay* is in use, the other slot should be left
Aside from the FMC and its carrier, the following hardware/software components
are required:
* For the PCI version: a standard PC with at least one free 4x (or wider)
PCI-Express slot.
* For the VME version: a VME64x crate with a MEN A20/A25 CPU.
* 50-ohm cables with 1-pin LEMO 00 plugs for connecting the I/O signals.
* Any Linux distribution (validated with
CentOS 7 3.10.0-957.1.3.rt56.913.el7.x86_64)
Modes of Operation
*fine-delay* can work in one or more of the following modes:
Pulse Delay
It produces one or more pulse(s) on selected outputs
a given time after an input trigger pulse.
Pulse Generator
Itproduces one or more pulse(s) on selected outputs
starting at an absolute time value programmed by the user.
In this mode, time base is usually provided by the White Rabbit network.
Time to Digital Converter
It tags all trigger pulses and delivers the timestamps to the user's
.. image:: drawings/func.png
:alt: *fine-delay* operating modes.
Modes (pulse delay/generator) can be selected independently for each output.
For example, one can configure the output 1 to delay trigger pulses
by 1 us, and the output 2 to produce a pulse at the beginning of each second.
The TDC mode can be enabled for the input at any time and
does not interfere with the operation of the channels being time tagged.
.. image:: drawings/front_panels.png
:alt: *fine-delay* front panel connector layout.
**Mechanical and environmental specs:**
* Format: FMC (VITA 57), with rear zone for conduction cooling.
* Operating temperature range: 0 - 90 degC.
* Carrier connection: 160-pin Low Pin Count FMC connector.
* 1 trigger input (LEMO 00).
* 4 pulse outputs (LEMO 00).
* 2 LEDs (termination status and trigger indicator).
* Carrier communication via 160-pin Low Pin Count FMC connector.
Trigger input
* TTL/LVTTL levels, DC-coupled. Reception of a trigger pulse is indicated by
blinking the "TRIG" LED in the front panel.
* 2 kOhm or 50 Ohm input impedance (programmable via software).
50 Ohm termination is indicated by the "TERM" LED in the front panel.
* Power-up input impedance: 2 kOhm.
* Protected against short circuit, overcurrent (> 200 mA) and overvoltage
(up to +28 V).
* Maximum input pulse edge rise time: 20 ns.
* TTL-compatible levels DC-coupled: Voh = 3 V, Vol = 200 mV (50 Ohm load),
Voh = 6 V, Vol = 400 mV (high impedance).
* Output impedance: 50 Ohm (source-terminated).
* Rise/fall time: 2.5 ns (10%% - 90%%, 50 Ohm load).
* Power-up state: LOW (2 kOhm pulldown), guaranteed glitch-free.
* Protected against continuous short circuit, overcurrent and overvoltage
(up to +28 V).
Power supply
* Used power supplies: P12V0, P3V3, P3V3_AUX, VADJ (voltage monitor only).
* Typical current consumption: 200 mA (P12V0) + 1.5 A (P3V3).
* Power dissipation: 7 W. Forced cooling is required.
.. image:: drawings/io_timing.png
:alt: *fine-delay* timing parameter definitions.
Time base
* On-board oscillator accuracy: +/- 2.5 ppm (i.e. max. 2.5 ns error for a
delay of 1 ms).
* When using White Rabbit as the timing reference: depending on the
characteristics of the grandmaster clock and the carrier used. On SPEC
v 4.0 FMC carrier, the accuracy is better than 1 ns.
Input timing
* Minimum pulse width: :math:`t_{IW}` = 50 ns. Pulses below 24 ns are rejected.
* Minimum gap between the last delayed output pulse and subsequent trigger
pulse: :math:`T_{LT}` = 50 ns.
* Input TDC performance: 400 ps pp accuracy, 27 ps resolution,
70 ps trigger-to-trigger rms jitter (measured at 500 kHz pulse rate).
Output timing
* Resolution: 10 ps.
* Accuracy (pulse generator mode): 300 ps.
* Train generation: trains of 1-65536 pulses or continuous square wave up
to 10 MHz.
* Output-to-output jitter (outputs programmed to the same delay): 10 ps rms.
* Output-to-output jitter (outputs programmed to to different delays, worst
case): 30 ps rms.
* Output pulse spacing (:math:`T_{SP}`) : 100 ns - 16 s. Adjustable in 10 ps
steps when both :math:`T_{PW}`, :math:`T_{GAP}` > 200 ns. Outside that range,
:math:`T_{SP}` resolution is limited to 4 ns.
* Output pulse start (:math:`t_{START}`) resolution: 10 ps for the rising edge
of the pulse, 10 ps for subsequent pulses if the condition above is met,
otherwise 4 ns.
Delay mode specific parameters
* Delay accuracy: < 1 ns.
* Trigger-to-output jitter: 80 ps rms.
* Trigger-to-output delay: minimum :math:`T_{DLY}` = 600 ns,
maximum :math:`T_{DLY}` = 120 s.
* Maximum trigger pulse rate: :math:`T_{DLY} + N*(T_{SP} + T_{GAP}) +` 100 ns,
where N = number of output pulses.
* Trigger pulses are ignored until the output with the biggest delay has
finished generation of the pulse(s).
Principles of Operation
.. note::
if you are an electronics engineer, you can skip this section, as