wrpc.tex

\documentclass[a4paper, 12pt]{article}
\usepackage{fullpage}
% % New line height: 1.05 * 1.2 = 1.26
\renewcommand{\baselinestretch}{1.05}

% To be able to use '\-/' in place of '-' inside \code{}
% so that long function names containing hyphens
% can be broken up after the hyphen:
\usepackage[shortcuts]{extdash}

% So that file names with multiple dots don't confuse
% graphicx package when using \includegraphics command:
\usepackage[multidot]{grffile}
\usepackage{graphicx}

\usepackage[usenames,dvipsnames,x11names]{xcolor}
\usepackage{amsmath}

% Workaround to fix mismatched left and right math delimiters. Taken from:
% http://tex.stackexchange.com/questions/63410/parentheses-differ-xelatex-fontspec-newtxmath-libertine
\DeclareSymbolFont{parenthesis}{T1}{fxl}{m}{n}
\DeclareMathDelimiter{(}{\mathopen}{parenthesis}{"28}{largesymbols}{"00}
\DeclareMathDelimiter{)}{\mathclose}{parenthesis}{"29}{largesymbols}{"01}
\DeclareMathDelimiter{[}{\mathopen}{parenthesis}{"5B}{largesymbols}{"02}
\DeclareMathDelimiter{]}{\mathclose}{parenthesis}{"5D}{largesymbols}{"03}
\DeclareMathDelimiter{\lbrace}{\mathopen}{parenthesis}{"7B}{largesymbols}{"08}
\DeclareMathDelimiter{\rbrace}{\mathclose}{parenthesis}{"7D}{largesymbols}{"09}

\usepackage{fancyvrb}
\usepackage{fancyhdr}
\pagestyle{plain}
\usepackage[final]{pdfpages} % inserts pages from a pdf file

\usepackage{titlesec}  % to change the appearance of section titles
\usepackage{listings}  % for syntax highlighted code listings
\usepackage{verbatim}  % for simple verbatim and comment environments
\usepackage{enumerate} % allows customized labels in enumerations
\usepackage{hyperref}  % makes cross references and URLs clickable
\definecolor{LinkRed}{HTML}{80171F}
\hypersetup{
  pdfauthor={Harold Abelson, Gerald Jay Sussman, Julie Sussman},
  pdftitle={Structure and Interpretation of Computer Programs, 2nd ed.},
  pdfsubject={computer science, programming, abstraction},
  colorlinks=true,
  linkcolor=LinkRed,
  urlcolor=LinkRed,
}

% Document colors
\definecolor{SchemeLight}   {HTML} {686868}
\definecolor{SchemeSteel}   {HTML} {888888}
\definecolor{SchemeDark}    {HTML} {262626}
\definecolor{SchemeBlue}    {HTML} {4172A3}
\definecolor{SchemeGreen}   {HTML} {487818}
\definecolor{SchemeBrown}   {HTML} {A07040}
\definecolor{SchemeRed}     {HTML} {AD4D3A}
\definecolor{SchemeViolet}  {HTML} {7040A0}
\definecolor{DropCapGray}   {HTML} {B8B8B8}
\definecolor{ChapterGray}   {HTML} {C8C8C8}
\definecolor{ChapterViolet} {HTML} {AEAECE}
\definecolor{DropCapViolet} {HTML} {9090C0}

\usepackage{lettrine}  % adds commands that make drop capitals
\renewcommand{\LettrineFontHook}{\rmfamily\mdseries\color{DropCapViolet}}
\renewcommand{\DefaultLraise}{0.00}
\renewcommand{\DefaultLoversize}{0.02}
\renewcommand{\DefaultLhang}{0.12}
\setlength{\DefaultFindent}{1pt}
\setlength{\DefaultNindent}{0em}

\lstset{%
  % Scheme syntax highlighter
    columns=fixed,
    extendedchars=true,
    upquote=true,
    showstringspaces=false,
    sensitive=false,
    mathescape=true,
    escapechar=~,
    alsodigit={>,<,/,-,=,!,?,*},
    alsoletter=',
    morestring=[b]",
    morecomment=[l];,
    % Keyword list taken form functional.py in Pygments package:
    morekeywords={lambda, define, if, else, cond, and, or, case,%
      let, let*, letrec, begin, do, delay, set!, =>, quote,%
      quasiquote, unquote, unquote-splicing, define-syntax, let-syntax,%
      letrec-syntax, syntax-rules},
    % If keywords are quoted, they must not be highlighted:
    emph={'lambda, 'define, 'if, 'else, 'cond, 'and, 'or, 'case,%
      'let, 'let*, 'letrec, 'begin, 'do, 'delay, 'set!, '=>, 'quote,%
      'quasiquote, 'unquote, 'unquote-splicing, 'define-syntax, 'let-syntax,%
      'letrec-syntax, 'syntax-rules},
    emphstyle=\color{SchemeDark},
    % Paint error red:
    emph={[2]error},emphstyle=[2]\color{SchemeRed},%
    % Builtins taken from functional.py:
    emph={[3]*, +, -, /, <, <=, =, >, >=, abs, acos, angle,
        append, apply, asin, assoc, assq, assv, atan,
        boolean?, caaaar, caaadr, caaar, caadar, caaddr, caadr,
        caar, cadaar, cadadr, cadar, caddar, cadddr, caddr,
        cadr, call-with-current-continuation, call-with-input-file,
        call-with-output-file, call-with-values, call/cc, car,
        cdaaar, cdaadr, cdaar, cdadar, cdaddr, cdadr, cdar,
        cddaar, cddadr, cddar, cdddar, cddddr, cdddr, cddr,
        cdr, ceiling, char->integer, char-alphabetic?, char-ci<=?,
        char-ci<?, char-ci=?, char-ci>=?, char-ci>?, char-downcase,
        char-lower-case?, char-numeric?, char-ready?, char-upcase,
        char-upper-case?, char-whitespace?, char<=?, char<?, char=?,
        char>=?, char>?, char?, close-input-port, close-output-port,
        complex?, cons, cos, current-input-port, current-output-port,
        denominator, display, dynamic-wind, eof-object?, eq?,
        equal?, eqv?, eval, even?, exact->inexact, exact?, exp,
        expt, floor, for-each, force, gcd, imag-part,
        inexact->exact, inexact?, input-port?, integer->char,
        integer?, interaction-environment, lcm, length, list,
        list->string, list->vector, list-ref, list-tail, list?,
        load, log, magnitude, make-polar, make-rectangular,
        make-string, make-vector, map, max, member, memq, memv,
        min, modulo, negative?, newline, not, null-environment,
        null?, number->string, number?, numerator, odd?,
        open-input-file, open-output-file, output-port?, pair?,
        peek-char, port?, positive?, procedure?, quotient,
        rational?, rationalize, read, read-char, real-part, real?,
        remainder, reverse, round, scheme-report-environment,
        set-car!, set-cdr!, sin, sqrt, string, string->list,
        string->number, string->symbol, string-append, string-ci<=?,
        string-ci<?, string-ci=?, string-ci>=?, string-ci>?,
        string-copy, string-fill!, string-length, string-ref,
        string-set!, string<=?, string<?, string=?, string>=?,
        string>?, string?, substring, symbol->string, symbol?,
        tan, transcript-off, transcript-on, truncate, values,
        vector, vector->list, vector-fill!, vector-length,
        vector-ref, vector-set!, vector?, with-input-from-file,
        with-output-to-file, write, write-char, zero?},
    emphstyle=[3]\color{SchemeViolet},%
    %
    basicstyle=\color{SchemeLight}\ttfamily,
    keywordstyle=\color{SchemeBlue}\bfseries,
    identifierstyle=\color{SchemeDark},
    stringstyle=\color{SchemeGreen},
    commentstyle=\color{SchemeLight}\itshape,
}

\newcommand{\acronym}[1]{\textsc{\fontspec[Numbers={OldStyle}]{Linux Libertine O}\MakeLowercase{#1}}}
\newcommand{\uppersc}[1]{{\fontspec[Letters=UppercaseSmallCaps]{Linux Libertine O}#1}}
\newcommand{\newterm}[1]{\index{#1}\emph{#1}}
\newcommand{\strong}[1]{\textbf{#1}}
\newcommand{\var}[1]{\textsl{#1}}
\newcommand{\code}[1]{\texttt{#1}}
\newcommand{\link}[1]{\hyperref[#1]{#1}}
\newcommand{\heading}[1]{{\sffamily\bfseries #1}}
\newcommand{\dark}{\color{SchemeDark}}
\newcommand{\mono}[1]{\hbox{\ttfamily\scriptsize #1}}
\newcommand{\monoit}[1]{\hbox{\ttfamily\itshape\scriptsize #1}}

\newenvironment{example}%
  {\verbatim\small}%
  {\endverbatim}

\newenvironment{smallexample}%
  {\verbatim\footnotesize}%
  {\endverbatim}

\lstnewenvironment{scheme}[1][]
{\lstset{basicstyle=\ttfamily\small\color{SchemeLight},#1}}
{}

\lstnewenvironment{smallscheme}[1][]
{\lstset{basicstyle=\ttfamily\footnotesize\color{SchemeLight},#1}}
{}

\makeindex
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% new stuff
\usepackage[overload]{textcase}
\newcommand{\codeHook}[1]{\mbox{\ttfamily\MakeTextUppercase{#1}}}
\usepackage{listings}
\usepackage{color}
\definecolor{light-gray}{gray}{0.95}
\usepackage{textcomp}
\lstset{upquote=true,  frame=single, captionpos=b, caption=, basicstyle=\scriptsize, backgroundcolor=\color{light-gray}, label=lst:init_src}

\usepackage{longtable} % table over many pages
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\begin{document}

\title{White Rabbit PTP Core User's Manual}
\author{Grzegorz Daniluk\\ CERN BE-CO-HT}


\begin{center}
	{\huge {\bf White Rabbit PTP Core User's Manual}}\\[0.2 cm]
	{\large CERN BE-CO-HT}\\[0.1 cm]
\end{center}



\abstract

This is the user manual for the White Rabbit PTP Core, part of the White
Rabbit project. It describes the building and running process. If you don't
want to get your hands dirty and prefer to use the binaries available at
\url{http://www.ohwr.org/projects/wr-cores/files} please skip
\link{Building the Core} and move forward directly to
\link{Running and Configuring}.

\newpage

% ##########################################################################
\label{Software and hardware requirements}
\section{Software and hardware requirements}

% ==========================================================================
\label{Repositories and Releases}
\subsection{Repositories and Releases}

This manual is about the official \textcolor{red}{TODO: release} stable release of the White
Rabbit PTP Core (\codeHook{WRPC}).

The code and documentation for the project is distributed in the
following places:

\begin{itemize}

\item \url{http://www.ohwr.org/projects/wr-cores/documents}

	hosts the pdf documentation for every official release.

\item \url{http://www.ohwr.org/projects/wr-cores/files}

	place where you can find a synthesized bitstream, ready to be downloaded to
  SPEC, for every stable release

\item \url{git://ohwr.org/hdl-core-lib/wr-cores.git}

	read-only repository with complete HDL sources of the \codeHook{WRPC}

\item \url{git://ohwr.org/hdl-core-lib/wr-cores/wrpc-sw.git}

  read-only repository with the \codeHook{WRPC} \codeHook{LM32} software

\end{itemize}
Other tools useful for building and running the \codeHook{WRPC} can be downloaded from
the following locations:

\begin{itemize}

\item \url{git://ohwr.org/misc/hdl-make.git}

  \textit{hdlmake} is used in the HDL synthesis process to create a Makefile based
on the set of Manifest files.

\item \url{http://www.ohwr.org/attachments/download/1133/lm32.tar.xz}

  \codeHook{LM32} toolchain used to compile the \codeHook{WRPC} software

\end{itemize}
Repositories containing the \codeHook{WRPC} gateware and software (\textit{wr-cores},
\textit{wrpc-sw}) are tagged with \codeHook{wrpc-v3.0} tag. Other tools
used to build the core and load it into \codeHook{SPEC} board should be used in their
newest stable releases.


% ==========================================================================
\label{Required hardware}
\subsection{Required hardware}

The absolute minimum to run the \codeHook{wr ptp core} is a PC computer with
Linux and a Simple PCIe \codeHook{fmc} Carrier
(\codeHook{spec}) - \url{http://www.ohwr.org/projects/spec}. However, it is highly
recommended to use also the \codeHook{dio} \codeHook{fmc} card (\url{http://www.ohwr.org/projects/fmc-dio-5chttla})
to be able to feed 1-PPS and 10MHz from external clock and output 1-PPS aligned
to the WR time. To test the White Rabbit synchronization, you will also need:
\begin{itemize}
\item another \codeHook{spec} board with a \codeHook{dio} \codeHook{fmc} or a White Rabbit Switch;
\item pair of \codeHook{wr}-supported \codeHook{sfp} transceivers (the list of supported
\codeHook{sfp}s can be found on our wiki page \url{http://www.ohwr.org/projects/white-rabbit/wiki/SFP})
\item a roll of G652, single mode fiber to connect your \codeHook{spec}s or \codeHook{spec}
with a \codeHook{wr} Switch.
\end{itemize}

% ##########################################################################
\label{Building the Core}
\section{Building the Core}

\textbf{Note:} you can skip this chapter if you want to use the release binaries
available from \textit{ohwr.org}.

\vspace{1em}
Building the core is a two step process. First you have to
synthesize the FPGA firmware (gateware) and then compile the software which
will be executed by the \codeHook{lm32} soft-core processor. If you don't need to
modify the \codeHook{lm32} software, you can skip the compilation stage since
synthesized gateware already embeds the default software for the release.

% ==========================================================================
\label{HDL synthesis}
\subsection{HDL synthesis}

Before running the synthesis process you have to make sure your environment is
set up correctly. You need a Xilinx ISE software with at least a WebPack
license. \textit{ISE} provides a set of scripts: \textit{settings32.sh},
\textit{settings32.csh}, \textit{settings64.sh} and \textit{settings64.csh} that configure all
the system variables required by the Xilinx software. Depending on a shell you
use and whether your Linux is 32 or 64-bits you should execute one of them
before the other tools are used. For 64-bit system and BASH shell you should
call:
\begin{lstlisting}
/opt/Xilinx/<version>/ISE_DS/settings64.sh
\end{lstlisting}

The easiest way to ensure that \textit{ISE}-related variables are always set in your
shell is adding the execution of the script to your \textit{bash.rc} file. You can
check if the shell is configured correctly by verifying if the \textit{\$XILINX}
variable contains path to your \textit{ISE} installation directory.

\textbf{Note:} current version of \textit{hdlmake} tool developed at CERN requires
modification of \textit{\$XILINX} variable after \textit{settings} script execution.
This (provided that the installation path for \textit{ISE} is /opt/Xilinx/<version>)
should be the following:

\begin{lstlisting}
# export XILINX=/opt/Xilinx/<version>/ISE_DS
\end{lstlisting}

\textbf{Note:} the Xilinx project file included in the \codeHook{wrpc} sources was created
with Xilinx ISE 14.5. It is however recommended to use the newest available
version of the ISE software.

\vspace{1em}
HDL sources for the \codeHook{wr ptp core} could be synthesized using Xilinx ISE without
any additional tools, but using \textit{hdlmake} is more convenient. It creates a
synthesis Makefile and ISE project file based on a set of Manifest.py files
deployed among the directories inside the \textit{wr-cores} repository.

First, please clone the \textit{hdlmake} repository from its location given in
\link{Repositories and Releases}:
\begin{lstlisting}
# wget http://www.ohwr.org/attachments/download/2070/hdlmake-v1.0
# git clone git://ohwr.org/misc/hdl-make.git <your_location>/hdl-make
# cd <your_location>/hdl-make
# git checkout 9d303ee
\end{lstlisting}

Then, using your favorite editor, you should create an \textit{hdlmake} script in
/usr/bin to be able to call it from any directory. The script should have the
following content:
\begin{lstlisting}
#!/usr/bin/env bash
python2.7 /path_to_hdlmake_sources/hdl-make/hdlmake/__main__.py #@
\end{lstlisting}

After that, you should make your script executable:
\begin{lstlisting}
chmod a+x /usr/bin/hdlmake
\end{lstlisting}

\vspace{1em}
Having Xilinx ISE software and \textit{hdlmake} in place, you can clone the main
\codeHook{wr ptp core} git repository and start building the FPGA bitstream.
First, please create a local copy of the \textit{wr-cores}:
\begin{lstlisting}
# git clone git://ohwr.org/hdl-core-lib/wr-cores.git <your_location>/wr-cores
# cd <your_location>/wr-cores
\end{lstlisting}

To build the gateware using sources of a stable release wrpc-v3.0, you
have to checkout the proper git tag:

\begin{lstlisting}
# git checkout wrpc-v3.0
\end{lstlisting}

If you use \textit{wr-cores} within another project (like \textit{wr-nic}), you may need
to check out another release tag for this repository. Please refer to the
project's documentation to find out which version of this package you need to
build.

You also need to fetch other git repositories containing modules instantiated
inside the \codeHook{wr ptp core} HDL. They are configured as git submodules:
\begin{lstlisting}
# git submodule init
# git submodule update
\end{lstlisting}

The local copies of the submodules are stored to:

\begin{lstlisting}
<your_location>/wr-cores/ip_cores
\end{lstlisting}

\vspace{1em}
The subdirectory which contains the main synthesis Manifest.py for \codeHook{spec} board
and in which you should perform the whole process is:

\begin{lstlisting}
# cd <your_location>/wr-cores/syn/spec_1_1/wr_core_demo/
\end{lstlisting}

First, please call \textit{hdlmake} to create synthesis Makefile for Xilinx
ISE:

\begin{lstlisting}
# hdlmake
\end{lstlisting}

After that, the actual synthesis is just the matter of executing:

\begin{lstlisting}
# make
\end{lstlisting}

This takes (depending on your computer speed) about 15 minutes and should create
two files with FPGA firmware: \textit{spec\_top.bit} and \textit{spec\_top.bin}. The
former can be downloaded to FPGA with Xilinx Platform Cable using e.g.
\textit{Xilinx Impact}. The latter can be used with kernel drivers from the
\textit{spec-sw} repository (check example in \link{Running and Configuring}).

\vspace{1em}
If, on the other hand, you would like to clean-up the repository and rebuild
everything from scratch you can use the following commands:
\begin{itemize}
\item \textit{\# make clean} - removes all synthesis reports and log files;
\item \textit{\# make mrproper} - removes spec\_top.bin and spec\_top.bit files;
\end{itemize}

% ==========================================================================
\label{LM32 software compilation}
\subsection{LM32 software compilation}

\textbf{Note:} By default, the \codeHook{lm32} software for a stable release is embedded
inside the FPGA bitstream you've downloaded from \textit{ohwr.org} or synthesized in
the previous step. This means you don't have to do a manual compilation of the
\codeHook{lm32} software unless you want to use a development version or you've made
some changes required by your application.

\vspace{1em}
To compile the \codeHook{lm32} software for the White Rabbit \codeHook{ptp} Core you will
need to download and unpack the \codeHook{lm32} toolchain from the location mentioned
in \link{Repositories and Releases}:

\begin{lstlisting}
# wget http://www.ohwr.org/attachments/download/1133/lm32.tar.xz
# tar xJf lm32.tar.xz -C <your_lm32_location>
\end{lstlisting}

Then you need to set a \texttt{CROSS\_COMPILE} variable in order
to compile the software for the \codeHook{lm32} processor:

\begin{lstlisting}
# export CROSS_COMPILE="<your_lm32_location>/lm32/bin/lm32-elf-"
\end{lstlisting}

To get the sources of the \codeHook{wrpc} software please clone the \textit{wrpc-sw} git
repository tagged with wrpc-v3.0 tag. If you use \codeHook{wrpc} within another
project, you may need to checkout a different tag or a specific commit. If this
applies, please refer to a documentation for this project.

\begin{lstlisting}
# git clone git://ohwr.org/hdl-core-lib/wr-cores/wrpc-sw.git <your_location>/wrpc-sw
# cd <your_location>/wrpc-sw
# git checkout wrpc-v3.0   # or "git checkout master"
\end{lstlisting}

Before you can compile \textit{wrpc-sw} you need to make a few configuration choices.
The package uses \textit{Kconfig} as a configuration engine, so you may run one of the
following commnads (the first is text-mode, the second uses a KDE GUI
and the third uses a Gnome GUI):

\begin{lstlisting}
# make menuconfig
# make xconfig
# make gconfig
\end{lstlisting}

Other \textit{Kconfig} target applies, like \code{config}, \code{oldconfig}
and so on.  A few default known-good configurations are found in
\texttt{./configs} and you choose one by \textit{make}ing it by name:

\begin{lstlisting}
# make spec_defconfig
\end{lstlisting}

The most important configuration choice at this point in time is
whether to enable Etherbone or not. It is disabled by default in
\code{spec\_defconfig} and enabled by default in
\code{etherbone\_defconfig}.

After the package is configured, just run \code{make} without
parameters to build your binary file:

\begin{lstlisting}
# make
\end{lstlisting}

The first time you build, the \textit{Makefile} automatically downloads
the \textit{git submodules} of this package, unless you already did that
by hand. The second and later build won't download anything
from the network.

The resulting binary \textit{wrc.bin} can be then used with the loader from
\textit{spec-sw} software package to program the \codeHook{lm32} inside the White Rabbit \codeHook{ptp}
Core (\link{Running and Configuring}).

% ##########################################################################
\label{Running and Configuring}
\section{Running and Configuring}

% ==========================================================================
\label{Downloading firmware to SPEC}
\subsection{Downloading firmware to SPEC}

For this step you will need a \codeHook{spec} board software support package
(\codeHook{spec-sw}) from \textit{ohwr.org}. It is a set of Linux kernel drivers and
userspace tools, that interact with a \codeHook{spec} board plugged into PCI-Express
slot.

Instructions in this section are based on a development version of \codeHook{spec-sw}
so if a stable release more recent than \textit{2014-02} is available, you should
use it instead. 

If there is a more recent version of the \codeHook{spec} software support, the
up-to-date documentation can always be found in \textit{doc/} subdirectory of
\codeHook{spec-sw} git repository.

\vspace{1em}
First, please clone the git repository of \codeHook{spec-sw} package and build it:

\begin{lstlisting}
# git clone git://ohwr.org/fmc-projects/spec/spec-sw.git <your_specsw_location>
# cd <your_specsw_location>
# git checkout c0e18a7
# make
\end{lstlisting}

Then you should copy your \textit{spec\_top.bin} generated in \link{HDL synthesis} or
downloaded from the \textit{ohwr} to /lib/firmware/fmc/. changing its
name:

\textbf{Note:} the commands below have to be executed with superuser rights

\begin{lstlisting}
# sudo cp <your_location>/wr-cores/syn/spec_1_1/wr_core_demo/spec_top.bin \
     /lib/firmware/fmc/spec-3.0.bin
\end{lstlisting}

You have to download also the "golden" firmware for \codeHook{spec} card. It is used by
the drivers to recognize the hardware:

\begin{lstlisting}
# wget http://www.ohwr.org/attachments/download/4057/spec-init.bin-2015-09-18
# sudo mv spec-init.bin-2015-09-18 /lib/firmware/fmc/spec-init.bin
\end{lstlisting}

Now you can load the drivers necessary to access \codeHook{spec} board from your
system:
\begin{lstlisting}
# sudo insmod fmc-bus/kernel/fmc.ko
# sudo insmod kernel/spec.ko
\end{lstlisting}

By default, when loading the \textit{spec.ko} driver FPGA gets programmed with
the "golden" bitstream. Starting from version 3.0, \codeHook{wr ptp core} uses a flash
memory chip on the carrier as a default place for storing the calibration
parameters and the init script. Also the storage format of this information is
now better organised in the files of the \codeHook{sdbfs} filesystem. Therefore,
starting from v3.0 you have to write the \codeHook{sdbfs} filesystem image to the
flash before running the \codeHook{wr ptp core}. You can download the image from our
project page:
\begin{lstlisting}
# wget http://www.ohwr.org/attachments/download/4060/sdbfs-flash.bin
\end{lstlisting}
It contains all the files required by the \codeHook{wr ptp core}. They are empty, but
have to exist in the \codeHook{sdbfs} structure to be written later as described in
\link{Writing configuration}. To store the filesystem image in flash, please
execute the following command:
\begin{lstlisting}
# sudo tools/flash-write -b 0x20 -c 0x0 0 1507712 < \
     <your_location>/sdbfs-flash.bin
\end{lstlisting}

\textbf{Note:} The \code{-b} parameter takes the PCI bus address of the SPEC. This
can be found either in the list of the PCI devices installed in the system
(\code{lspci}) or in the kernel messages (\code{dmesg}) when inserting the
spec.ko module. See the example below on loading the FPGA code. We see that the
second number in the colon/dotted notation provides the argument in the example
above.

\textbf{Note:} Please refer to \link{Writing SDBFS image in standalone configuration}
for instructions on how to write the \codeHook{sdbfs} image to a standalone \codeHook{spec}
or custom hardware.

\vspace{1em}
Now, you are ready to load the last driver, which downloads the actual
\codeHook{wr ptp core} bitstream to the Spartan 6 FPGA:
\begin{lstlisting}
# sudo insmod fmc-bus/kernel/fmc-trivial.ko gateware=fmc/spec-3.0.bin
\end{lstlisting}

You can use the \textit{dmesg} Linux command to verify if the FPGA firmware file was
loaded into the FPGA. Among plenty of messages you should be able to find
something very similar to:

\begin{lstlisting}[basicstyle=\tiny]
[1275526.738895] spec 0000:20:00.0:  probe for device 0020:0000
[1275526.738906] spec 0000:20:00.0: PCI INT A -> GSI 16 (level, low) -> IRQ 16
[1275526.738913] spec 0000:20:00.0: setting latency timer to 64
[1275526.743102] spec 0000:20:00.0: got file "fmc/spec-init.bin", 1485236 (0x16a9b4) bytes
[1275526.934710] spec 0000:20:00.0: FPGA programming successful
[1275527.296754] spec 0000:20:00.0: mezzanine 0
[1275527.296756]       Manufacturer: CERN
[1275527.296757]       Product name: FmcDio5cha

[1275593.973147] fmc FmcDio5cha-2000: Driver has no ID: matches all
[1275593.973177] spec 0000:20:00.0: reprogramming with fmc/spec-3.0.bin
[1275594.168249] spec 0000:20:00.0: FPGA programming successful
\end{lstlisting}

If everything went right up to this moment you have your board running the FPGA 
bitstream with a default \codeHook{lm32} software. If you want to load your own
\textit{wrc.bin} built from the \textit{wrpc-sw} repository you can use the \textit{spec-cl}
tool:

\begin{lstlisting}
# sudo tools/spec-cl <your_location>/wrpc-sw/wrc.bin
\end{lstlisting}

\vspace{1em}
Now you should be able to start a Virtual-UART tool (also part of the
\codeHook{spec-sw} package) that will be used to interact with the \codeHook{wr ptp core}
shell:

\begin{lstlisting}
# sudo tools/spec-vuart
\end{lstlisting}

If you are able to see the \codeHook{wrpc} Shell prompt \textit{wrc\#} this means the Core
is up and running on your \codeHook{spec}. Congratulations !


% ==========================================================================
\label{Writing configuration}
\subsection{Writing configuration}

First, you should perform a few configuration steps through the \codeHook{wrpc} shell
before using the core.

\textbf{Note:} the examples below describe only a subset of the \codeHook{wrpc} Shell
commands. The full list of supported commands can be found in
\link{WRPC Shell commands}.

\vspace{1em}
Before making the configuration changes, it is good to stop the \codeHook{ptp} daemon.
Then, debug messages from the daemon will not show up to the console while you
interact with the shell.

\begin{lstlisting}
wrc# ptp stop
\end{lstlisting}

\vspace{1em}
First you should make sure your board has a proper \codeHook{mac} address assigned:
\begin{lstlisting}
wrc# mac get
\end{lstlisting}
If the result of above command is \texttt{MAC-address: 22:33:ww:xx:yy:zz}, this means
\codeHook{mac} was not yet configured and stored in the Flash/EEPROM. The value is
based on thermometer serial number as is unique among SPEC devices,
globally accepted as ``locally assigned'', but you might want to assign your own address. A value \texttt{22:33:44:55:66:77} is the final fallback if no
thermometer is found (very unlikely). You should get
the \codeHook{mac} for your board from its manufacturer. To configure the address and
store it into the Flash/EEPROM (so that it's automatically loaded every time the
\codeHook{wrpc} starts) you should type two commands in the \codeHook{wrpc} shell:
\begin{lstlisting}
wrc# mac set xx:xx:xx:xx:xx:xx
wrc# mac setp xx:xx:xx:xx:xx:xx
\end{lstlisting}
where \texttt{xx:xx:xx:xx:xx:xx} is the \codeHook{mac} address of your board.

\vspace{1em}
Next you should create a calibration database with fixed delays values and
alpha parameters. The example below presents the \codeHook{wrpc} Shell commands that
clear all previous entries and add two Axcen transceivers with deltaTx, deltaRx
and alpha parameters associated with them.

\begin{lstlisting}
wrc# sfp erase
wrc# sfp add AXGE-1254-0531 180625 148451 72169888
wrc# sfp add AXGE-3454-0531 180625 148451 -73685416
\end{lstlisting}

To check the content of the \codeHook{sfp} database you can execute the \textit{sfp show} shell
command.

\textbf{Note:} The deltaTx and deltaRx parameters above are the defaults for
wrpc-v3.0 release bitstream available on \textit{ohwr.org}, running on
\codeHook{spec} v4 board and calibrated to port 1 of a \codeHook{wr} Switch v3.3. These
values as well as the parameters for the \codeHook{wr} Switch are available on the
calibration wiki page
(\textit{http://www.ohwr.org/projects/white-rabbit/wiki/Calibration}). However, if
you re-synthesize the firmware or want to have the most accurate estimation of
the fixed delays and alpha for your fiber, you should read and perform the
\codeHook{wr} Calibration procedure (\textit{http://www.ohwr.org/documents/213}).

\vspace{1em}
The \codeHook{wr ptp core} mode of operation (GrandMaster/Master/Slave) can be set
using the \textit{mode} command:

\begin{lstlisting}
wrc# mode gm       # for GrandMaster mode
wrc# mode master   # for Master mode
wrc# mode slave    # for Slave mode
\end{lstlisting}

This stops the \codeHook{ptp} daemon, changes the mode of operation, but does not
start it automatically. Therefore, after calling it, you need to restart the
daemon manually:

\begin{lstlisting}
wrc# ptp start
\end{lstlisting}

\textbf{Note:} For running the GrandMaster mode, you need to provide 1-PPS and 10MHz
signal from an external source (e.g. GPS receiver or Cesium clock). Please
connect 1-PPS signal to the LEMO connector No.4 and 10MHz to the LEMO connector
No.5 on the \codeHook{fmc} \codeHook{dio} mezzanine board.

\vspace{1em}
One option is to type all the commands to initialize the \codeHook{wrpc} software to
the required state every time the Core starts. However, you can also write your
own initialization script to the Flash/EEPROM. It will be executed every time
the \codeHook{wrpc} software starts. A simple script that loads the calibration
parameters, configures the \codeHook{wr} mode to Slave and starts the \codeHook{ptp} daemon
is presented below:

\begin{lstlisting}
wrc# init erase
wrc# init add ptp stop
wrc# init add sfp match
wrc# init add mode slave
wrc# init add ptp start
\end{lstlisting}

Almost exactly the same one can be used for running \codeHook{wrpc} in the GrandMaster
or Master mode. The only difference would be changing the
\textit{init add mode slave} line to \textit{init add mode gm} or
\textit{init add mode master}.

% ==========================================================================
\label{Running the Core}
\subsection{Running the Core}

Having the \codeHook{sfp} database, and the init script created in \link{Writing
configuration} you can restart the \codeHook{wr ptp core} by reprogramming the
\codeHook{lm32} software (with \textit{spec-cl} tool) or by typing the shell command:

\begin{lstlisting}
wrc# init boot
\end{lstlisting}

You should see log messages that confirm the execution of the initialization
script:

\begin{lstlisting}
executing: ptp stop
PTP stop
executing: sfp match
AXGE-3454-0531
SFP matched, dTx=180707 dRx=148323 alpha=-73685416
executing: mode slave
PTP stop
Locking PLL
executing: ptp start
PTP start
Slave Only, clock class set to 255
\end{lstlisting}

Now you should have the \codeHook{wr ptp core} running in \codeHook{wr} Slave mode.
\codeHook{wrpc} needs to make a calibration of t24p phase transition value. It has to
be done only once for a new bitstream and is performed automatically when
\codeHook{wrpc} runs in the Slave mode. That is why it is very important, even if
\codeHook{wrpc} is meant to run in the Master mode, to configure it to Slave for a
moment and connect to any \codeHook{wr} Master. This has to be repeated every time
a new bitstream (gateware) is deployed. The measured value is automatically
stored to Flash/EEPROM and used later in the Master or GrandMaster mode.

The Shell also contains a monitoring function which you can use to check the
\codeHook{wr} synchronization status:

\begin{lstlisting}
wrc# gui
\end{lstlisting}

The information is presented in a clear, auto-refreshing screen. The
information is refreshed at every WR iteration or periodically if
nothing else happens (so you see an up-to-date timestamp). The period
defaults to 1 second and can be changed using the \textit{refresh} command. To exit from
this console mode press <Esc>. A full description of the information reported
by gui is provided in \link{WRPC GUI elements}.

\textbf{Note:} the \textit{Synchronization status} and \textit{Timing parameters} in \textit{gui}
are available only in the \codeHook{wr} Slave mode. When running as \codeHook{wr} Master, you
would be able to see only the current date and time, link status, Tx and Rx
packet counters, lock and calibration status.

\vspace{1em}
\includegraphics[width=12cm]{wrpc_mon.png}
\vspace{1em}

If you want to log statistics from the \codeHook{wrpc} operation, it's probably
better to use the \textit{stat} shell command. It reports the same information as GUI
but in a single long line, a form which is easier to parse and analyze:

\begin{lstlisting}
wrc# stat
lnk:1 rx:416 tx:118 lock:1 sv:1 ss:'TRACK_PHASE' aux:0 sec:94197 \
nsec:793068184 mu:836241 dms:400556 dtxm:10 drxm:163610 dtxs:0 drxs:128400 \
asym:35129 crtt:544221 cko:-5 setp:7667 hd:61479 md:37221 ad:65000 ucnt:101 \
temp: 45.6875 C
lnk:1 rx:417 tx:119 lock:1 sv:1 ss:'TRACK_PHASE' aux:0 sec:94198 \
nsec:293076296 mu:836253 dms:400562 dtxm:10 drxm:163610 dtxs:0 drxs:128400 \
asym:35129 crtt:544233 cko:-4 setp:7663 hd:61485 md:37259 ad:65000 ucnt:102 \
temp: 45.6875 C
(...)
\end{lstlisting}

\vspace{1em}
Unlike \textit{gui}, the \textit{stat} command runs asynchronously: you can still
issue shell commands while stats are running (this is different from
earlier \texttt{wrpc-sw} releases).  You can stop statistics by running \textit{stat} again.
As an alternative to the toggling action of \textit{stat} alone, you can
use ``\textit{stat 1}'' or ``\textit{stat 0}''.

Statistics are printed every time the WR servo runs; thus no statistics
are reported when the node is running in master mode, nor when your node
is running as slave and the master disappeared.

\vspace{1em}
If you have a \codeHook{dio} mezzanine board plugged to your \codeHook{spec}, you can verify
the synchronization performance by observing the offset between 1-PPS signals
from the \codeHook{wr} Master and \codeHook{wr} Slave. The \codeHook{wr ptp core} generates 1-PPS
signal on the LEMO connector No. 1. Please remember to use oscilloscope cables
of the same length and type (with the same delay), or take their delay
difference into account in your measurements.


% ==========================================================================
\label{Diagnostics via SNMP}
\subsection{Diagnostics via SNMP}

Up to the version 3.1 of \codeHook{wrpc} the only way to perform diagnostics
of the \texttt{wrpc-sw} was to use serial console with tools like \textit{gui}, \textit{stat},
etc. For some set-ups, like standalone node, it is very inconvenient to use
external console for diagnostics.

From the version 3.1 of \codeHook{wrpc} it is possible to include the \textit{Mini
SNMP responder}, which allows to perform remote diagnostics using \textit{SNMP} via
a port connected to the \textit{Write Rabbit} network.

The configuration file of \codeHook{wrpc} contains the following
SNMP-related options:
\begin{itemize}
\item \texttt{CONFIG\_SNMP} -- include the \textit{Mini SNMP responder} into \codeHook{wrpc}
\item \texttt{CONFIG\_SNMP\_SET} -- enable the support of SNMP \textit{SET} packets
\item \texttt{CONFIG\_SNMP\_VERBOSE} -- enable verbose output from the \textit{Mini SNMP
      responder} on the \codeHook{wrpc}'s console
\end{itemize}

The MIB file describing \codeHook{wrpc}'s OIDs can be found in the \texttt{lib} directory
of the \texttt{wrpc-sw} repo.

So far, the \textit{Mini SNMP responder} supports version 1 and a subset of version
2c of the SNMP protocol.

The following types of requests are supported:
\begin{itemize}
   \item GET -- get value of a given OID
   \item GETNEXT -- get value of a next OID after the given OID (this is used
         for \texttt{snmpwalk}s)
   \item SET -- change the value of a given OID (so far used only for adding
         SFP's to the database and PTP restarts)
\end{itemize}

The \textit{Mini SNMP responder} does not support:
\begin{itemize}
   \item bulk requests packets (GETBULK)
   \item more than one OID in the request packet
   \item \texttt{trap} and \texttt{inform} packets
   \item encryption
   \item authentication
   \item SNMPv2c return error types; all returned error types follows SNMPv1
\end{itemize}

To make examples more readable, listings below use \texttt{SNMP\_OPT} environment
variable. Make sure you set it properly in your shell.
\begin{lstlisting}
 # SNMP_OPT="-c public -v 2c -m WR-WRPC-MIB -M +/var/lib/mibs/ietf:lib \
   192.168.1.20"
\end{lstlisting}
where:
\begin{itemize}
   \item \texttt{-c public} -- sets SNMP community as "\textit{public}"
   \item \texttt{-v 2c} -- specifies SNMP version
   \item \texttt{-m WR-WRPC-MIB} -- specifies MIBs to be loaded
   \item \texttt{-M +/var/lib/mibs/ietf:lib} -- contains path to MIBs in the host
         system (\texttt{/var/lib/mibs/ietf}) and path to \texttt{WR-WRPC-MIB} (\texttt{lib});
         on Debian-like systems default MIBs can be downloaded using
         \texttt{download-mibs} command (package \texttt{snmp-mibs-downloader}); on
         CentOS and RedHat MIBs are included in the \texttt{libsmi} package
   \item \texttt{192.168.1.20} -- the IP address of the target board
\end{itemize}

For example, to get the system uptime please execute the \texttt{snmpget} command:
\begin{lstlisting}
 # snmpget #SNMP_OPT wrpcTimeSystemUptime.0
\end{lstlisting}

To get dump of all available OIDs please execute the \texttt{snmpwalk} command:
\begin{lstlisting}
 # snmpwalk #SNMP_OPT wrpcCore
\end{lstlisting}
Part of the \texttt{snmpwalk}'s output:
\begin{lstlisting}
 WR-WRPC-MIB::wrpcVersionHwType.0 = STRING: spec
 WR-WRPC-MIB::wrpcVersionSwVersion.0 = STRING: wrpc-v3.0-251-g14e952e
 WR-WRPC-MIB::wrpcVersionSwBuildBy.0 = STRING: Adam Wujek
 WR-WRPC-MIB::wrpcVersionSwBuildDate.0 = STRING: Jun  7 2016 18:12:24
 WR-WRPC-MIB::wrpcTimeTAI.0 = Counter64: 1465375022
 WR-WRPC-MIB::wrpcTimeTAIString.0 = STRING: 2016-06-08-08:37:02
 WR-WRPC-MIB::wrpcTimeSystemUptime.0 = Timeticks: (18186) 0:03:01.86
 WR-WRPC-MIB::wrpcTemperatureName.1 = STRING: pcb
 WR-WRPC-MIB::wrpcTemperatureValue.1 = STRING: 38.5625
 WR-WRPC-MIB::wrpcSpllMode.0 = INTEGER: slave(3)
 WR-WRPC-MIB::wrpcSpllIrqCnt.0 = Counter32: 1259605
 [...]
 WR-WRPC-MIB::wrpcPortSfpInDB.0 = INTEGER: inDataBase(2)
 WR-WRPC-MIB::wrpcPortInternalTx.0 = Counter32: 452
 WR-WRPC-MIB::wrpcPortInternalRx.0 = Counter32: 869
 WR-WRPC-MIB::wrpcSfpPn.1 = STRING: AXGE-1254-0531
 WR-WRPC-MIB::wrpcSfpDeltaTx.1 = INTEGER: 180625
 WR-WRPC-MIB::wrpcSfpDeltaRx.1 = INTEGER: 148451
 WR-WRPC-MIB::wrpcSfpAlpha.1 = INTEGER: 72169888
 End of MIB
\end{lstlisting}

It is recommended to use SNMP v2c for communication with the \codeHook{wrpc}.
Please note that when the version 1 of SNMP is used, 64 bit counters are not
supported. This makes it impossible to read some \codeHook{wrpc}'s objects with
SNMPv1.

% --------------------------------------------------------------------------
\label{Managing SFP database via SNMP}
\subsubsection{Managing SFP database via SNMP}

The SFPs database can be displayed using the \texttt{sfp show} command from
the \codeHook{wrpc}'s console:
\begin{lstlisting}
 wrc# sfp show
 1: PN:AXGE-1254-0531   dTx:   180625 dRx:   148451 alpha: 72169888
 2: PN:AXGE-3454-0531   dTx:   180625 dRx:   148451 alpha: -73685416
\end{lstlisting}

The same data is exported by the \textit{Mini SNMP responder} via the table
\texttt{wrpcSfpTable}:

\begin{lstlisting}
 # snmpwalk #SNMP_OPT wrpcSfpTable
 WR-WRPC-MIB::wrpcSfpPn.1 = STRING: AXGE-1254-0531
 WR-WRPC-MIB::wrpcSfpPn.2 = STRING: AXGE-3454-0531
 WR-WRPC-MIB::wrpcSfpDeltaTx.1 = INTEGER: 180625
 WR-WRPC-MIB::wrpcSfpDeltaTx.2 = INTEGER: 180625
 WR-WRPC-MIB::wrpcSfpDeltaRx.1 = INTEGER: 148451
 WR-WRPC-MIB::wrpcSfpDeltaRx.2 = INTEGER: 148451
 WR-WRPC-MIB::wrpcSfpAlpha.1 = INTEGER: 72169888
 WR-WRPC-MIB::wrpcSfpAlpha.2 = INTEGER: -73685416
 End of MIB
\end{lstlisting}

When the SET support is compiled into the \textit{Mini SNMP responder}, it is
possible to erase or add/replace SFP entires to the SFPs database via SNMP.

Addition (or modification) of one SFP to the database can done by a row of
SNMP SETs. Firstly, please set the delta Tx (\texttt{wrpcPtpConfigDeltaTx.0}), the
delta Rx (\texttt{wrpcPtpConfigDeltaRx.0}) and the alpha (\texttt{wrpcPtpConfigAlpha.0})
with new values.
Then, to commit the change to the SFP database, perform the SNMP SET on
the \texttt{wrpcPtpConfigApply.0} with the value \texttt{writeToFlashCurrentSfp}. It will
add/update values for the currently plugged SFP.

To add/update entries for different SFPs, please set deltas and alpha like
above, then set PN of an SFP to the \texttt{wrpcPtpConfigSfpPn.0} and commit
the change by setting \texttt{writeToFlashGivenSfp} to the \texttt{wrpcPtpConfigApply.0}.

It is also possible to update values in the memory for the current SFP.
For that, please set delta Tx, delta Rx and alpha as described above,
then set \texttt{writeToMemoryCurrentSfp} to the \texttt{wrpcPtpConfigApply.0}