Skip to content

W
White Rabbit Trigger Distribution
Commit 704000a2 authored by Federico Vaga's avatar Federico Vaga
Browse files
Options

sw:fw:fd: FineDelay initialization 

This code has been ported form the fine-delay-sw as is. Probably it will
need some review to reduce code size.
Signed-off-by: Tomasz Wlostowski's avatarTomasz Wlostowski <tomasz.wlostowski@cern.ch>
Signed-off-by: Federico Vaga's avatarFederico Vaga <federico.vaga@cern.ch>
parent b7da873f
Hide whitespace changes
Inline Side-by-side
Showing with 1379 additions and 2 deletions
software/firmware/fd/Makefile
View file @ 704000a2
-include Makefile.specific
OBJS := wrtd-rt-fd.o
OBJS += acam.o gpio.o pll.o i2c.o calibrate.o fine-delay-init.o # Fine-Delay init
OBJS += common/wrtd-fw-common.o
OBJS += common/loop-queue.o
OBJDIR += common
......@@ -8,8 +9,8 @@ OUTPUT = wrtd-rt-fd
TRTL ?= ../../../dependencies/mock-turtle/
TRTL_SW = $(TRTL)/software
EXTRA_CFLAGS += -I../../include
EXTRA_CFLAGS += -I../common
EXTRA_CFLAGS += -I../../include
EXTRA_CFLAGS += -I../common -Werror
vpath %.c ../
......
software/firmware/fd/acam.c 0 → 100644
View file @ 704000a2
/*
* Accessing the ACAM chip and configuring it.
*
* Copyright (C) 2012 CERN (www.cern.ch)
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
* Author: Alessandro Rubini <rubini@gnudd.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* version 2 as published by the Free Software Foundation or, at your
* option, any later version.
*/
#include "mockturtle-rt.h"
#include "hw/fd_main_regs.h"
#include "hw/acam_gpx.h"
#include "fine-delay-init.h"
static int fd_acam_addr = -1;
int fd_bin_size = -1;
/*
* Calculation is fixed point: picoseconds and 16 decimals (i.e. ps << 16).
* We know the bin is small, but the Tref is several nanos so we need 64 bits
* (although our current values fit in 32 bits after the division)
*/
#define ACAM_FP_BIN ((int)(ACAM_DESIRED_BIN * (1 << 16)))
#define ACAM_FP_TREF (((1000LL * 1000 * 1000) << 16) / ACAM_CLOCK_FREQ_KHZ)
/* Default values of control registers for the ACAM TDC working in G-Mode
(eeprom values are obsolete) */
#define ACAM_GMODE_START_OFFSET 10000
#define ACAM_GMODE_ASOR 17000
#define ACAM_GMODE_ATMCR (26 | (1500 << 8))
#define ACAM_GMODE_ADSFR 84977
static int acam_calc_pll(uint64_t tref, int bin, int *hsdiv_out,
int *refdiv_out)
{
uint64_t tmpll;
int x, refdiv, hsdiv;
/*
* Tbin(I-mode) = (Tref << refdiv) / (216 * hsdiv)
*
* so, calling X the value "hsdiv >> refdiv" we have
*
* X = Tref / (216 * Tbin)
*
* Then, we can choose refdiv == 7 to have the best bits,
* and then shift out the zeros to get smaller values.
*
*/
if (0) {
x = (tref << 16) / 216 / bin;
//printf("x = %lf\n", (double)x / (1<<16));
} else {
/* We can't divide 64 bits in kernel space */
tmpll = div_u64(tref << 16, 216);
x = div_u64(tmpll, bin);
}
/* Now, shift out the max bits (usually 7) and drop decimal part */
refdiv = ACAM_MAX_REFDIV;
hsdiv = (x << refdiv) >> 16;
/* Check the first decimal bit and approximate */
if ((x << refdiv) & (1 << 15))
hsdiv++;
/* until we have zeroes as LSB, shift out to decrease pll quotient */
while (refdiv > 0 && !(hsdiv & 1)) {
refdiv--;
hsdiv >>= 1;
}
*hsdiv_out = hsdiv;
*refdiv_out = refdiv;
/* Finally, calculate what we really have */
if (0) {
bin = (tref << refdiv) / 216 / hsdiv;
} else {
tmpll = div_u64(tref << refdiv, 216);
bin = div_u64(tmpll, hsdiv);
}
return (bin + 1); /* We always return the bin size in the I mode. Other modes should scale it appropriately. */
}
static void acam_set_address(int addr)
{
if (addr == fd_acam_addr)
return;
if (fd_acam_addr == -1) {
/* first time */
fd_gpio_dir(0xf00, FD_GPIO_OUT);
}
fd_gpio_val(0xf00, addr << 8);
fd_acam_addr = addr;
}
/* Warning: acam_readl and acam_writel only work if GCR.BYPASS is set */
uint32_t acam_readl(int reg)
{
acam_set_address(reg);
fd_writel(FD_TDCSR_READ, FD_REG_TDCSR);
return fd_readl(FD_REG_TDR) & ACAM_MASK;
}
void acam_writel(int val, int reg)
{
acam_set_address(reg);
fd_writel(val, FD_REG_TDR);
fd_writel(FD_TDCSR_WRITE, FD_REG_TDCSR);
}
static void acam_set_bypass(int on)
{
/* warning: this clears the "input enable" bit: call at init only */
fd_writel(on ? FD_GCR_BYPASS : 0, FD_REG_GCR);
}
static inline int acam_is_pll_locked()
{
return !(acam_readl(12) &AR12_NotLocked);
}
/* Two test functions to verify the bus is working -- Tom */
static int acam_test_addr_bit(int base, int bit,
int data)
{
int addr1 = base;
int addr2 = base + (1<<bit);
int reg;
reg = acam_readl(addr1) & ~data;
acam_writel(reg, addr1); /* zero the data mask */
reg = acam_readl(addr2) | data;
acam_writel(reg, addr2); /* set the data mask */
if ((acam_readl(addr1) & data) != 0)
goto out;
if ((acam_readl(addr2) & data) != data)
goto out;
/* the other way around */
reg = acam_readl(addr2) & ~data;
acam_writel(reg, addr2); /* zero the data mask */
reg = acam_readl(addr1) | data;
acam_writel(reg, addr1); /* set the data mask */
if ((acam_readl(addr2) & data) != 0)
goto out;
if ((acam_readl(addr1) & data) != data)
goto out;
return 0;
out:
pr_error("ACAM address bit %d failure\n", bit);
return -EIO;
}
static int acam_test_bus()
{
int err = 0, i, v;
/* Use register 5 to checke the data bits */
for(i = 0; i & ACAM_MASK; i <<= 1) {
acam_writel(i, 5);
acam_readl(0);
v = acam_readl(5);
if (v != i)
goto out;
acam_writel(~i & ACAM_MASK, 5);
acam_readl( 0);
v = acam_readl( 5);
if (v != (~i & ACAM_MASK))
goto out;
}
err += acam_test_addr_bit(0, 0, 0x000001);
err += acam_test_addr_bit(1, 1, 0x000008);
err += acam_test_addr_bit(0, 2, 0x000001);
err += acam_test_addr_bit(3, 3, 0x010000);
if (err)
return -EIO;
return 0;
out:
pp_printf("Error: ACAM data bit 0x%06x failure\n", i);
return -EIO;
}
/* We need to write come static configuration in the registers */
struct acam_init_data {
int addr;
int val;
};
/* Commented values are not constant, they are added at runtime (see later) */
static struct acam_init_data acam_init_regs_gmode[] = {
{0, AR0_ROsc | AR0_RiseEn0 | AR0_RiseEn1 | AR0_HQSel},
{1, AR1_Adj(0, 0) | AR1_Adj(1, 0) | AR1_Adj(2, 5) |
AR1_Adj(3, 0) | AR1_Adj(4, 5) | AR1_Adj(5, 0) | AR1_Adj(6, 5)},
{2, AR2_GMode | AR2_Adj(7, 0) | AR2_Adj(8, 5) |
AR2_DelRise1(0) | AR2_DelFall1(0) | AR2_DelRise2(0) | AR2_DelFall2(0)},
{3, AR3_DelTx(1,3) | AR3_DelTx(2,3) | AR3_DelTx(3,3) | AR3_DelTx(4,3) |
AR3_DelTx(5,3) | AR3_DelTx(6,3) | AR3_DelTx(7,3) | AR3_DelTx(8,3) |
AR3_RaSpeed(0,3) | AR3_RaSpeed(1,3) | AR3_RaSpeed(2,3)},
{4, AR4_EFlagHiZN | AR4_RaSpeed(3,3) | AR4_RaSpeed(4,3) |
AR4_RaSpeed(5,3) | AR4_RaSpeed(6,3) | AR4_RaSpeed(7,3) | AR4_RaSpeed(8,3)},
{5, AR5_StartRetrig
| 0 /* AR5_StartOff1(hw->calib.acam_start_offset) */
| AR5_MasterAluTrig},
{6, AR6_Fill(200) | AR6_PowerOnECL},
{7, /* AR7_HSDiv(hsdiv) | AR7_RefClkDiv(refdiv) */ 0
| AR7_ResAdj | AR7_NegPhase},
{11, 0x7ff0000},
{12, 0x0000000},
{14, 0},
/* finally, reset */
{4, AR4_EFlagHiZN | AR4_MasterReset | AR4_StartTimer(0)},
};
static struct acam_init_data acam_init_regs_imode[] = {
{0, AR0_TRiseEn(0) | AR0_HQSel | AR0_ROsc},
{2, AR2_IMode},
{5, AR5_StartOff1(3000) | AR5_MasterAluTrig},
{6, 0},
{7, /* AR7_HSDiv(hsdiv) | AR7_RefClkDiv(refdiv) */ 0
| AR7_ResAdj | AR7_NegPhase},
{11, 0x7ff0000},
{12, 0x0000000},
{14, 0},
/* finally, reset */
{4, AR4_EFlagHiZN | AR4_MasterReset | AR4_StartTimer(0)},
};
static int acam_configure( enum fd_acam_modes mode, struct acam_init_data *regs, int n_regs )
{
int i, hsdiv, refdiv, reg7val;
struct acam_init_data *p;
uint32_t regval;
int locked = 0;
fd_bin_size = acam_calc_pll(ACAM_FP_TREF, ACAM_FP_BIN, &hsdiv, &refdiv);
reg7val = AR7_HSDiv(hsdiv) | AR7_RefClkDiv(refdiv);
pr_debug("ACAM config: mode %d bin 0x%x, hsdiv %i, refdiv %i\n", mode, fd_bin_size, hsdiv, refdiv);
/* Disable TDC inputs prior to configuring */
fd_writel(FD_TDCSR_STOP_DIS | FD_TDCSR_START_DIS, FD_REG_TDCSR);
/* Disable the ACAM PLL for a while to make sure it is reset */
acam_writel(0, 0);
acam_writel(7, 0);
mdelay(100);
for (p = regs, i = 0; i < n_regs; p++, i++) {
regval = p->val;
if (p->addr == 7)
regval |= reg7val;
if (p->addr == 5 && mode == ACAM_GMODE )
regval |= AR5_StartOff1(ACAM_GMODE_START_OFFSET);
if (p->addr == 6 && mode == ACAM_GMODE )
regval |= AR6_StartOff2(ACAM_GMODE_START_OFFSET);
acam_writel(regval, p->addr);
}
for (i = 0; i < 20; i++)
{
if (acam_is_pll_locked())
{
locked = 1;
break;
}
mdelay(100);
}
if(!locked)
{
pp_printf("Error: ACAM PLL doesn't lock.\n");
return -1;
}
/* after config, set the FIFO address for further reads */
acam_set_address(8);
return 0;
}
int fd_acam_init()
{
int ret;
fd_acam_addr = -1; /* First time must be activated */
acam_set_bypass(1); /* Driven by host, not core */
if ( (ret = acam_test_bus()) )
return ret;
if ( (ret = acam_configure(ACAM_IMODE, acam_init_regs_imode, ARRAY_SIZE(acam_init_regs_imode))) )
return ret;
if ( (ret = fd_calibrate_outputs()) )
return ret;
if ( (ret = acam_configure(ACAM_GMODE, acam_init_regs_gmode, ARRAY_SIZE(acam_init_regs_gmode))) )
return ret;
acam_set_bypass(0); /* Driven by core, not host */
/* Clear and disable the timestamp readout buffer */
fd_writel(FD_TSBCR_PURGE | FD_TSBCR_RST_SEQ, FD_REG_TSBCR);
/*
* Program the ACAM-specific TS registers w pre-defined calib values:
* - bin -> internal timebase scalefactor (ADSFR),
* - Start offset (must be consistent with value in ACAM reg 4)
* - timestamp merging control register (ATMCR)
* GMode fix: we no longer use the values from the EEPROM (they are fixed anyway)
*/
fd_writel( ACAM_GMODE_ADSFR, FD_REG_ADSFR);
fd_writel( ACAM_GMODE_ASOR, FD_REG_ASOR);
fd_writel( ACAM_GMODE_ATMCR, FD_REG_ATMCR);
return 0;
}
software/firmware/fd/calibrate.c 0 → 100644
View file @ 704000a2
/*
* Calibrate the output path.
*
* Copyright (C) 2012 CERN (www.cern.ch)
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
* Author: Alessandro Rubini <rubini@gnudd.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* version 2 as published by the Free Software Foundation or, at your
* option, any later version.
*/
#include "mockturtle-rt.h"
#include "fine-delay-init.h"
#include "hw/fd_main_regs.h"
#include "hw/acam_gpx.h"
#include "hw/fd_channel_regs.h"
/* This is the same as in ./acam.c: use only at init time */
static void acam_set_bypass( int on)
{
fd_writel( on ? FD_GCR_BYPASS : 0, FD_REG_GCR);
}
/*
* Measures the the FPGA-generated TDC start and the output of one of
* the fine delay chips (channel) at a pre-defined number of taps
* (fine). Retuns the delay in picoseconds. The measurement is
* repeated and averaged (n_avgs) times. Also, the standard deviation
* of the result can be written to (sdev) if it's not NULL.
*/
/* Note: channel is the "internal" one: 0..3 */
static uint64_t output_delay_ps(int ch, int fine, int n)
{
int i;
uint64_t acc = 0;
/* Disable the output for the channel being calibrated */
fd_gpio_clr( FD_GPIO_OUTPUT_EN(FD_CH_EXT(ch)));
/* Enable the stop input in ACAM for the channel being calibrated */
acam_writel( AR0_TRiseEn(0) | AR0_TRiseEn(FD_CH_EXT(ch))
| AR0_HQSel | AR0_ROsc, 0);
/* Program the output delay line setpoint */
fd_drv_ch_writel( ch, fine, FD_REG_FRR);
fd_drv_ch_writel( ch, FD_DCR_ENABLE | FD_DCR_MODE | FD_DCR_UPDATE,
FD_REG_DCR);
fd_drv_ch_writel( ch, FD_DCR_FORCE_DLY | FD_DCR_ENABLE, FD_REG_DCR);
/*
* Set the calibration pulse mask to genrate calibration
* pulses only on one channel at a time. This minimizes the
* crosstalk in the output buffer which can severely decrease
* the accuracy of calibration measurements
*/
fd_writel( FD_CALR_PSEL_W(1 << ch), FD_REG_CALR);
udelay(1);
/* Do n_avgs single measurements and average */
for (i = 0; i < n; i++) {
uint32_t fr;
/* Re-arm the ACAM (it's working in a single-shot mode) */
fd_writel( FD_TDCSR_ALUTRIG, FD_REG_TDCSR);
udelay(1);
/* Produce a calib pulse on the TDC start and the output ch */
fd_writel( FD_CALR_CAL_PULSE |
FD_CALR_PSEL_W(1 << ch), FD_REG_CALR);
udelay(1);
/* read the tag, convert to picoseconds (fixed point: 16.16) */
fr = acam_readl( 8 /* fifo */) & 0x1ffff;
//pp_printf("i %d fr %x\n\r", i, fr);
acc += fr * fd_bin_size;
}
fd_drv_ch_writel(ch, 0, FD_REG_DCR);
/* Calculate avg, min max */
acc = div_u64((acc + n / 2), n );
//pp_printf("ch %d avg %08x%08x\n\r", ch, (uint32_t)(acc>>32), (uint32_t)acc);
return acc;
}
static int fd_find_8ns_tap(int ch)
{
int l = 0, mid, r = FD_NUM_TAPS - 1;
uint64_t bias, dly;
/*
* Measure the delay at zero setting, so it can be further
* subtracted to get only the delay part introduced by the
* delay line (ingoring the TDC, FPGA and routing delays).
* Use a binary search of the delay value.
*/
bias = output_delay_ps(ch, 0, FD_CAL_STEPS);
while( r - l > 1) {
mid = ( l + r) / 2;
dly = output_delay_ps(ch, mid, FD_CAL_STEPS) - bias;
if(dly < 8000 << 16)
l = mid;
else
r = mid;
}
return l;
}
/**
* fd_calibrate_outputs
* It calibrates the delay line by finding the correct 8ns-tap value
* for each channel. This is done during ACAM initialization, so on driver
* probe.
*/
int fd_calibrate_outputs()
{
int ch;
int measured;
acam_set_bypass(1); /* not useful */
fd_writel(FD_TDCSR_START_EN | FD_TDCSR_STOP_EN, FD_REG_TDCSR);
for (ch = FD_CH_1; ch <= FD_CH_LAST; ch++) {
measured = fd_find_8ns_tap(ch);
fd_drv_ch_writel(ch, measured, FD_REG_FRR);
pr_debug("Channel %d: 8ns @ %i taps.\n\r", ch+1, measured );
}
return 0;
}
software/firmware/fd/fine-delay-init.c 0 → 100644
View file @ 704000a2
/*
* Copyright (C) 2018 CERN (www.cern.ch)
* Author: Federico Vaga <federico.vaga@cern.ch>
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
*
* SPDX-License-Identifier: LGPL-3.0-or-later
*/
#include <errno.h>
#include <string.h>
#include "mockturtle-rt.h"
#include <mockturtle-framework.h>
#include "wrtd-common.h"
#include "wrtd-fw-common.h"
#include "hw/fd_channel_regs.h"
#include "hw/fd_main_regs.h"
#include "fine-delay-init.h"
#include "wrtd-fw-internals.h"
/**
* fd_do_reset
* The reset function (by Tomasz)
*
* This function can reset the entire mezzanine (FMC) or just
* the fine-delay core (CORE).
* In the reset register 0 means reset, 1 means normal operation.
*/
static void fd_do_reset(int hw_reset)
{
if (hw_reset) {
/* clear RSTS_RST_FMC bit, set RSTS_RST_CORE bit*/
fd_writel( FD_RSTR_LOCK_W(0xdead) | FD_RSTR_RST_CORE_MASK,
FD_REG_RSTR);
udelay(10000);
fd_writel( FD_RSTR_LOCK_W(0xdead) | FD_RSTR_RST_CORE_MASK
| FD_RSTR_RST_FMC_MASK, FD_REG_RSTR);
/* TPS3307 supervisor needs time to de-assert master reset */
mdelay(600);
return;
}
/* clear RSTS_RST_CORE bit, set RSTS_RST_FMC bit */
fd_writel( FD_RSTR_LOCK_W(0xdead) | FD_RSTR_RST_FMC_MASK,
FD_REG_RSTR);
udelay(1000);
fd_writel( FD_RSTR_LOCK_W(0xdead) | FD_RSTR_RST_FMC_MASK
| FD_RSTR_RST_CORE_MASK, FD_REG_RSTR);
udelay(1000);
}
/* Some init procedures to be intermixed with subsystems */
int fd_gpio_defaults()
{
fd_gpio_dir( FD_GPIO_TRIG_INTERNAL, FD_GPIO_OUT);
fd_gpio_set( FD_GPIO_TRIG_INTERNAL);
fd_gpio_set( FD_GPIO_OUTPUT_MASK);
fd_gpio_dir( FD_GPIO_OUTPUT_MASK, FD_GPIO_OUT);
fd_gpio_dir( FD_GPIO_TERM_EN, FD_GPIO_OUT);
fd_gpio_clr( FD_GPIO_TERM_EN);
return 0;
}
int fd_reset_again()
{
/* Reset the FD core once we have proper reference/TDC clocks */
fd_do_reset( 0 /* not hw */);
mdelay(10);
if (! ( fd_readl(FD_REG_GCR) & FD_GCR_DDR_LOCKED) )
{
pr_error("timeout waiting for GCR lock bit\n");
return -EIO;
}
fd_do_reset( 0 /* not hw */);
return 0;
}
/* FIXME missing all calibration */
int fd_init(void)
{
int err, ch;
pr_debug("Initializing the Fine Delay board...\n");
fd_do_reset(1);
err = fd_gpio_init();
if (err)
return err;
err = fd_pll_init();
if (err)
return err;
fd_gpio_defaults();
err = fd_reset_again();
if (err)
return err;
err = fd_acam_init();
if (err)
return err;
for (ch = 1; ch <= FD_CH_NUMBER; ch++)
fd_gpio_set( FD_GPIO_OUTPUT_EN(ch));
// todo: read offsets from the EEPROM. I2C should be working OK.
return 0;
}
software/firmware/fd/fine-delay-init.h 0 → 100644
View file @ 704000a2
#ifndef __FINE_DELAY_WRAPPER_H
#define __FINE_DELAY_WRAPPER_H
extern int fd_bin_size;
/* Channels are called 1..4 in all docs. Internally it's 0..3 */
#define FD_CH_1 0
#define FD_CH_LAST 3
#define FD_CH_NUMBER 4
#define FD_CH_INT(i) ((i) - 1)
#define FD_CH_EXT(i) ((i) + 1)
#define FD_NUM_TAPS 1024 /* This is an hardware feature of SY89295U */
#define FD_CAL_STEPS 1024 /* This is a parameter: must be power of 2 */
#define FD_MAGIC_FPGA 0xf19ede1a /* FD_REG_IDR content */
/* Values for the configuration of the acam PLL. Can be changed */
#define ACAM_DESIRED_BIN 80.9553
#define ACAM_CLOCK_FREQ_KHZ 31250
/* ACAM TDC operation modes */
enum fd_acam_modes {
ACAM_RMODE,
ACAM_IMODE,
ACAM_GMODE
};
#define FD_GPIO_TERM_EN 0x0001 /* Input terminator enable */
#define FD_GPIO_OUTPUT_EN(x) \
(1 << (6-(x))) /* Output driver enable */
#define FD_GPIO_OUTPUT_MASK 0x003c /* Output driver enable */
#define FD_GPIO_TRIG_INTERNAL 0x0040 /* TDC trig (1=in, 1=fpga) */
#define FD_GPIO_CAL_DISABLE 0x0080 /* 0 enables calibration */
/*
* You can change the following value to have a pll with smaller divisor,
* at the cost of potentially less precision in the desired bin value.
*/
#define ACAM_MAX_REFDIV 7
#define ACAM_MASK ((1<<29) - 1) /* 28 bits */
/* SPI Bus chip selects */
#define FD_CS_DAC 0 /* DAC for VCXO */
#define FD_CS_PLL 1 /* AD9516 PLL */
#define FD_CS_GPIO 2 /* MCP23S17 GPIO */
/* MCP23S17 register addresses (only ones which are used by the lib) */
#define FD_MCP_IODIR 0x00
#define FD_MCP_IPOL 0x01
#define FD_MCP_IOCON 0x0a
#define FD_MCP_GPIO 0x12
#define FD_MCP_OLAT 0x14
int fd_spi_xfer(int ss, int num_bits,
uint32_t in, uint32_t *out);
int fd_spi_init();
int fd_pll_init();
int fd_acam_init();
uint32_t acam_readl(int reg);
void acam_writel(int val, int reg);
/* Functions exported by calibrate.c, called within acam.c */
int fd_calibrate_outputs();
/* Functions exported by gpio.c */
int fd_gpio_init();
void fd_gpio_exit();
void fd_gpio_dir(int pin, int dir);
void fd_gpio_val(int pin, int val);
void fd_gpio_set_clr(int pin, int set);
int fd_init();
#define fd_gpio_set(pin) fd_gpio_set_clr((pin), 1)
#define fd_gpio_clr(pin) fd_gpio_set_clr((pin), 0)
#define FD_GPIO_IN 0
#define FD_GPIO_OUT 1
#define DP_BASE_FD_CORE 0x0
static inline void fd_writel(uint32_t val, uint32_t reg)
{
dp_writel( val, reg + DP_BASE_FD_CORE );
}
static inline uint32_t fd_readl(uint32_t reg)
{
return dp_readl( reg + DP_BASE_FD_CORE );
}
static inline uint32_t fd_drv_ch_readl(int ch,
unsigned long reg)
{
return fd_readl( 0x100 + ch * 0x100 + reg);
}
static inline void fd_drv_ch_writel(int ch,
uint32_t v, unsigned long reg)
{
fd_writel( v, 0x100 + ch * 0x100 + reg);
}
// Alessandro, I f*****ing hate you. And all kernel devs. To express my hatred I will use
// C++ style comments intermixed with C style comments!
static inline uint64_t div_u64(uint64_t dividend, uint32_t divisor)
{
return dividend / divisor;
}
#endif
\ No newline at end of file
software/firmware/fd/gpio.c 0 → 100644
View file @ 704000a2
/*
* SPI access to fine-delay internals
*
* Copyright (C) 2012 CERN (www.cern.ch)
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
* Author: Alessandro Rubini <rubini@gnudd.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* version 2 as published by the Free Software Foundation or, at your
* option, any later version.
*/
#include "mockturtle-rt.h"
#include "fine-delay-init.h"
static uint32_t fd_mcp_iodir = 0;
static uint32_t fd_mcp_olat = 0;
#define SPI_RETRIES 100
int gpio_writel(int val, int reg)
{
int rval = fd_spi_xfer( FD_CS_GPIO, 24,
0x4e0000 | (reg << 8) | val, NULL);
return rval;
}
static int gpio_readl(int reg)
{
uint32_t ret;
int err;
err = fd_spi_xfer(FD_CS_GPIO, 24,
0x4f0000 | (reg << 8), &ret);
if (err < 0)
return err;
return ret & 0xff;
}
static int gpio_writel_with_retry( int val, int reg)
{
int retries = SPI_RETRIES, rv;
while(retries--)
{
gpio_writel( val, reg);
rv = gpio_readl( reg);
if(rv >= 0 && (rv == val))
{
if(SPI_RETRIES-1-retries > 0)
pr_debug("gpio_writel_with_retry: succeded after %d retries\n",
SPI_RETRIES - 1 - retries);
return 0;
}
}
return -EIO;
}
void fd_gpio_dir(int mask, int dir)
{
fd_mcp_iodir &= ~mask;
if (dir == FD_GPIO_IN)
fd_mcp_iodir |= mask;
gpio_writel_with_retry( (fd_mcp_iodir & 0xff), FD_MCP_IODIR);
gpio_writel_with_retry( (fd_mcp_iodir >> 8), FD_MCP_IODIR+1);
}
void fd_gpio_val(int mask, int values)
{
fd_mcp_olat &= ~mask;
fd_mcp_olat |= values;
gpio_writel_with_retry( (fd_mcp_olat & 0xff), FD_MCP_OLAT);
gpio_writel_with_retry( (fd_mcp_olat >> 8), FD_MCP_OLAT+1);
}
void fd_gpio_set_clr(int mask, int set)
{
if (set)
fd_gpio_val(mask, mask);
else
fd_gpio_val(mask, 0);
}
int fd_gpio_init()
{
int i, val;
fd_mcp_iodir = 0xffff;
fd_mcp_olat = 0;
if (gpio_writel(0x00, FD_MCP_IOCON) < 0)
goto out;
/* Try to read and write a register to test the SPI connection */
for (val = 0xaa; val >= 0; val -= 0x11) {
if (gpio_writel(val, FD_MCP_IPOL) < 0)
goto out;
i = gpio_readl( FD_MCP_IPOL);
if (i < 0)
goto out;
if (i != val) {
pr_error("GPIO comm error (got 0x%x, expected 0x%x)\n", i, val);
return -EIO;
}
}
/* last time we wrote 0, ok */
return 0;
out:
return -EIO;
}
software/firmware/fd/hw/acam_gpx.h 0 → 100644
View file @ 704000a2
#ifndef __ACAM_GPX_H
#define __ACAM_GPX_H
#define AR0_ROsc (1<<0)
#define AR0_RiseEn0 (1<<1)
#define AR0_FallEn0 (1<<2)
#define AR0_RiseEn1 (1<<3)
#define AR0_FallEn1 (1<<4)
#define AR0_RiseEn2 (1<<5)
#define AR0_FallEn2 (1<<6)
#define AR0_HQSel (1<<7)
#define AR0_TRiseEn(port) (1<<(10+port))
#define AR0_TFallEn(port) (1<<(19+port))
#define AR1_Adj(chan, value) (((value) & 0xf) << (chan * 4))
#define AR2_GMode (1<<0)
#define AR2_IMode (1<<1)
#define AR2_RMode (1<<2)
#define AR2_Disable(chan) (1<<(3+chan))
#define AR2_Adj(chan, value) (((value)&0xf)<<(12+4*(chan-7)))
#define AR2_DelRise1(value) (((value)&0x3)<<(20))
#define AR2_DelFall1(value) (((value)&0x3)<<(22))
#define AR2_DelRise2(value) (((value)&0x3)<<(24))
#define AR2_DelFall2(value) (((value)&0x3)<<(26))
#define AR3_DelTx(chan, value) (((value)&0x3)<<(5 + (chan -1 ) * 2))
#define AR3_RaSpeed(chan, value) (((value)&0x3)<<(21 + (chan ) * 2))
#define AR4_RaSpeed(chan, value) (((value)&0x3)<<(10 + (chan-3) * 2))
#define AR3_Zero (0) // nothing interesting for the Fine Delay
#define AR4_StartTimer(value) ((value) & 0xff)
#define AR4_Quiet (1<<8)
#define AR4_MMode (1<<9)
#define AR4_MasterReset (1<<22)
#define AR4_PartialReset (1<<23)
#define AR4_AluTrigSoft (1<<24)
#define AR4_EFlagHiZN (1<<25)
#define AR4_MTimerStart (1<<26)
#define AR4_MTimerStop (1<<27)
#define AR5_StartOff1(value) ((value)&0x3ffff)
#define AR5_StopDisStart (1<<21)
#define AR5_StartDisStart (1<<22)
#define AR5_MasterAluTrig (1<<23)
#define AR5_PartialAluTrig (1<<24)
#define AR5_MasterOenTrig (1<<25)
#define AR5_PartialOenTrig (1<<26)
#define AR5_StartRetrig (1<<27)
#define AR6_Fill(value) ((value)&0xff)
#define AR6_StartOff2(value) (((value)&0x3ffff)<<8)
#define AR6_InSelECL (1<<26)
#define AR6_PowerOnECL (1<<27)
#define AR7_HSDiv(value) ((value)&0xff)
#define AR7_RefClkDiv(value) (((value)&0x7)<<8)
#define AR7_ResAdj (1<<11)
#define AR7_NegPhase (1<<12)
#define AR7_Track (1<<13)
#define AR7_MTimer(value) (((value) & 0x1ff)<<15)
#define AR14_16BitMode (1<<4)
#define AR8I_IFIFO1(reg) ((reg) & 0x1ffff)
#define AR8I_Slope1(reg) ((reg) & (1<<17) ? 1 : 0)
#define AR8I_StartN1(reg) (((reg) >> 18) & 0xff)
#define AR8I_ChaCode1(reg) (((reg) >> 26) & 0x3)
#define AR9I_IFIFO2(reg) ((reg) & 0x1ffff)
#define AR9I_Slope2(reg) ((reg) & (1<<17) ? 1 : 0)
#define AR9I_StartN2(reg) (((reg) >> 18) & 0xff)
#define AR9I_ChaCode2(reg) (((reg) >> 26) & 0x3)
#define AR8R_IFIFO1(reg) ((reg) & 0x3fffff)
#define AR9R_IFIFO2(reg) ((reg) & 0x3fffff)
#define AR11_StopCounter0(num) ((num) & 0xff)
#define AR11_StopCounter1(num) (((num) & 0xff) << 8)
#define AR11_HFifoErrU(num) (1 << (num+16))
#define AR11_IFifoErrU(num) (1 << (num+24))
#define AR11_NotLockErrU (1 << 26)
#define AR12_HFifoE (1<<11)
#define AR12_NotLocked (1<<10)
#endif
software/firmware/fd/hw/pll_config.h 0 → 100644
View file @ 704000a2
struct ad9516_reg {
int reg;
int val;
};
const struct ad9516_reg __9516_regs[] = {
{0x0000, 0x99}, /* Config SPI */
{0x0001, 0x00},
{0x0002, 0x10},
{0x0003, 0xC3},
{0x0004, 0x00},
/* PLL */
{0x0010, 0x7C}, /* PFD and charge pump */
{0x0011, 0x05}, /* R divider (1) */
{0x0012, 0x00}, /* R divider (2) */
{0x0013, 0x0C}, /* A counter */
{0x0014, 0x12}, /* B counter (1) */
{0x0015, 0x00}, /* B counter (2) */
{0x0016, 0x05}, /* PLL control (1) */
{0x0017, 0xb4}, /* PLL control (2) PLL_STATUS = Lock Detect */
{0x0018, 0x07}, /* PLL control (3) */
{0x0019, 0x00}, /* PLL control (4) */
{0x001A, 0x00}, /* PLL control (5) */
{0x001B, 0xE0}, /* PLL control (6) */
{0x001C, 0x02}, /* PLL control (7) */
{0x001D, 0x00}, /* PLL control (8) */
{0x001E, 0x00}, /* PLL control (9) */
{0x001F, 0x0E}, /* PLL readback */
/* Fine Delay */
{0x00A0, 0x01}, /* OUT6 Delay bypass */
{0x00A1, 0x00}, /* OUT6 Delay full-scale */
{0x00A2, 0x00}, /* OUT6 Delay fraction */
{0x00A3, 0x01}, /* OUT7 Delay bypass */
{0x00A4, 0x00}, /* OUT7 Delay full-scale */
{0x00A5, 0x00}, /* OUT7 Delay fraction */
{0x00A6, 0x01}, /* OUT8 Delay bypass */
{0x00A7, 0x00}, /* OUT8 Delay full-scale */
{0x00A8, 0x00}, /* OUT8 Delay fraction */
{0x00A9, 0x01}, /* OUT9 Delay bypass */
{0x00AA, 0x00}, /* OUT9 Delay full-scale */
{0x00AB, 0x00}, /* OUT9 Delay fraction */
/* LVPECL */
{0x00F0, 0x08}, /* OUT0 */
{0x00F1, 0x08}, /* OUT1 */
{0x00F2, 0x08}, /* OUT2 */
{0x00F3, 0x18}, /* OUT3, inverted */
{0x00F4, 0x00}, /* OUT4 */
{0x00F5, 0x08}, /* OUT5 */
/* LVDS/CMOS */
{0x0140, 0x5A}, /* OUT6 */
{0x0141, 0x5A}, /* OUT7 */
{0x0142, 0x5B}, /* OUT8 */
{0x0143, 0x42}, /* OUT9 */
/* LVPECL Channel divider */
{0x0190, 0x00}, /* Divider 0 (1) */
{0x0191, 0x80}, /* Divider 0 (2) */
{0x0192, 0x00}, /* Divider 0 (3) */
{0x0193, 0x00}, /* Divider 1 (1) */
{0x0194, 0x80}, /* Divider 1 (2) */
{0x0195, 0x00}, /* Divider 1 (3) */
{0x0196, 0xFF}, /* Divider 2 (1) */
{0x0197, 0x00}, /* Divider 2 (2) */
{0x0198, 0x00}, /* Divider 2 (3) */
/* LVDS/CMOS Channel divider */
{0x0199, 0x33}, /* Divider 3 (1) */
{0x019A, 0x00}, /* Divider 3 (2) */
{0x019B, 0x11}, /* Divider 3 (3) */
{0x019C, 0x20}, /* Divider 3 (4) */
{0x019D, 0x00}, /* Divider 3 (5) */
{0x019E, 0x00}, /* Divider 4 (1) */
{0x019F, 0x00}, /* Divider 4 (2) */
{0x01A0, 0x11}, /* Divider 4 (3) */
{0x01A1, 0x20}, /* Divider 4 (4) */
{0x01A2, 0x00}, /* Divider 4 (5) */
{0x01A3, 0x00},
/* VCO Divider and CLK Input */
{0x01E0, 0x04}, /* VCO divider VCODIV = 6 */
{0x01E1, 0x02}, /* Input Clock */
/* System */
{0x0230, 0x00}, /* Power down and sync */
{0x0231, 0x00},
/* Update All registers */
{0x0232, 0x00}, /* Update All registers */
};
software/firmware/fd/i2c.c 0 → 100644
View file @ 704000a2
/*
* I2C access (on-board EEPROM)
*
* Copyright (C) 2012 CERN (www.cern.ch)
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
* Author: Alessandro Rubini <rubini@gnudd.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* version 2 as published by the Free Software Foundation or, at your
* option, any later version.
*/
#include "mockturtle-rt.h"
#include "fine-delay-init.h"
#include "hw/fd_main_regs.h"
static void set_sda(int val)
{
uint32_t reg;
reg = fd_readl(FD_REG_I2CR) & ~FD_I2CR_SDA_OUT;
if (val)
reg |= FD_I2CR_SDA_OUT;
fd_writel(reg, FD_REG_I2CR);
udelay(3);
}
static void set_scl(int val)
{
uint32_t reg;
reg = fd_readl(FD_REG_I2CR) & ~FD_I2CR_SCL_OUT;
if (val)
reg |= FD_I2CR_SCL_OUT;
fd_writel(reg, FD_REG_I2CR);
udelay(3);
}
static int get_sda()
{
return fd_readl(FD_REG_I2CR) & FD_I2CR_SDA_IN ? 1 : 0;
};
static void mi2c_start()
{
set_sda(0);
set_scl(0);
}
static void mi2c_stop()
{
set_sda(0);
set_scl(1);
set_sda(1);
}
int mi2c_put_byte(int data)
{
int i;
int ack;
for (i = 0; i < 8; i++, data<<=1) {
set_sda(data & 0x80);
set_scl(1);
set_scl(0);
}
set_sda(1);
set_scl(1);
ack = get_sda();
set_scl(0);
set_sda(0);
return ack ? -EIO : 0; /* ack low == success */
}
int mi2c_get_byte(unsigned char *data, int sendack)
{
int i;
int indata = 0;
/* assert: scl is low */
set_scl(0);
set_sda(1);
for (i = 0; i < 8; i++) {
set_scl(1);
indata <<= 1;
if (get_sda())
indata |= 0x01;
set_scl(0);
}
set_sda((sendack ? 0 : 1));
set_scl(1);
set_scl(0);
set_sda(0);
*data= indata;
return 0;
}
void mi2c_init()
{
set_scl(1);
set_sda(1);
}
void mi2c_scan()
{
int i;
for(i = 0; i < 256; i += 2) {
mi2c_start();
if(!mi2c_put_byte(i))
pr_debug("Found i2c device at 0x%x\n",
i >> 1);
mi2c_stop();
}
}
/* FIXME: this is very inefficient: read several bytes in a row instead */
int fd_eeprom_read(int i2c_addr, uint32_t offset,
void *buf, size_t size)
{
int i;
uint8_t *buf8 = buf;
unsigned char c;
for(i = 0; i < size; i++) {
mi2c_start();
if(mi2c_put_byte(i2c_addr << 1) < 0) {
mi2c_stop();
return -EIO;
}
mi2c_put_byte((offset >> 8) & 0xff);
mi2c_put_byte(offset & 0xff);
offset++;
mi2c_stop();
mi2c_start();
mi2c_put_byte((i2c_addr << 1) | 1);
mi2c_get_byte(&c, 0);
*buf8++ = c;
mi2c_stop();
}
return size;
}
software/firmware/fd/pll.c 0 → 100644
View file @ 704000a2
/*
* Copyright (C) 2018 CERN (www.cern.ch)
* Author: Federico Vaga <federico.vaga@cern.ch>
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
*
* SPDX-License-Identifier: LGPL-3.0-or-later
*/
#include <errno.h>
#include <string.h>
#include "mockturtle-rt.h"
#include <mockturtle-framework.h>
#include "wrtd-common.h"
#include "wrtd-fw-common.h"
#include "hw/fd_channel_regs.h"
#include "hw/fd_main_regs.h"
#include "fine-delay-init.h"
#include "wrtd-fw-internals.h"
struct ad9516_reg {
int reg;
int val;
};
const struct ad9516_reg __9516_regs[] = {
{0x0000, 0x99}, /* Config SPI */
{0x0001, 0x00},
{0x0002, 0x10},
{0x0003, 0xC3},
{0x0004, 0x00},
/* PLL */
{0x0010, 0x7C}, /* PFD and charge pump */
{0x0011, 0x05}, /* R divider (1) */
{0x0012, 0x00}, /* R divider (2) */
{0x0013, 0x0C}, /* A counter */
{0x0014, 0x12}, /* B counter (1) */
{0x0015, 0x00}, /* B counter (2) */
{0x0016, 0x05}, /* PLL control (1) */
{0x0017, 0xb4}, /* PLL control (2) PLL_STATUS = Lock Detect */
{0x0018, 0x07}, /* PLL control (3) */
{0x0019, 0x00}, /* PLL control (4) */
{0x001A, 0x00}, /* PLL control (5) */
{0x001B, 0xE0}, /* PLL control (6) */
{0x001C, 0x02}, /* PLL control (7) */
{0x001D, 0x00}, /* PLL control (8) */
{0x001E, 0x00}, /* PLL control (9) */
{0x001F, 0x0E}, /* PLL readback */
/* Fine Delay */
{0x00A0, 0x01}, /* OUT6 Delay bypass */
{0x00A1, 0x00}, /* OUT6 Delay full-scale */
{0x00A2, 0x00}, /* OUT6 Delay fraction */
{0x00A3, 0x01}, /* OUT7 Delay bypass */
{0x00A4, 0x00}, /* OUT7 Delay full-scale */
{0x00A5, 0x00}, /* OUT7 Delay fraction */
{0x00A6, 0x01}, /* OUT8 Delay bypass */
{0x00A7, 0x00}, /* OUT8 Delay full-scale */
{0x00A8, 0x00}, /* OUT8 Delay fraction */
{0x00A9, 0x01}, /* OUT9 Delay bypass */
{0x00AA, 0x00}, /* OUT9 Delay full-scale */
{0x00AB, 0x00}, /* OUT9 Delay fraction */
/* LVPECL */
{0x00F0, 0x08}, /* OUT0 */
{0x00F1, 0x08}, /* OUT1 */
{0x00F2, 0x08}, /* OUT2 */
{0x00F3, 0x18}, /* OUT3, inverted */
{0x00F4, 0x00}, /* OUT4 */
{0x00F5, 0x08}, /* OUT5 */
/* LVDS/CMOS */
{0x0140, 0x5A}, /* OUT6 */
{0x0141, 0x5A}, /* OUT7 */
{0x0142, 0x5B}, /* OUT8 */
{0x0143, 0x42}, /* OUT9 */
/* LVPECL Channel divider */
{0x0190, 0x00}, /* Divider 0 (1) */
{0x0191, 0x80}, /* Divider 0 (2) */
{0x0192, 0x00}, /* Divider 0 (3) */
{0x0193, 0x00}, /* Divider 1 (1) */
{0x0194, 0x80}, /* Divider 1 (2) */
{0x0195, 0x00}, /* Divider 1 (3) */
{0x0196, 0xFF}, /* Divider 2 (1) */
{0x0197, 0x00}, /* Divider 2 (2) */
{0x0198, 0x00}, /* Divider 2 (3) */
/* LVDS/CMOS Channel divider */
{0x0199, 0x33}, /* Divider 3 (1) */
{0x019A, 0x00}, /* Divider 3 (2) */
{0x019B, 0x11}, /* Divider 3 (3) */
{0x019C, 0x20}, /* Divider 3 (4) */
{0x019D, 0x00}, /* Divider 3 (5) */
{0x019E, 0x00}, /* Divider 4 (1) */
{0x019F, 0x00}, /* Divider 4 (2) */
{0x01A0, 0x11}, /* Divider 4 (3) */
{0x01A1, 0x20}, /* Divider 4 (4) */
{0x01A2, 0x00}, /* Divider 4 (5) */
{0x01A3, 0x00},
/* VCO Divider and CLK Input */
{0x01E0, 0x04}, /* VCO divider VCODIV = 6 */
{0x01E1, 0x02}, /* Input Clock */
/* System */
{0x0230, 0x00}, /* Power down and sync */
{0x0231, 0x00},
/* Update All registers */
{0x0232, 0x00}, /* Update All registers */
};
int fd_spi_xfer(int ss, int num_bits, uint32_t in, uint32_t *out)
{
uint32_t scr = 0, r;
unsigned long timeout;
scr = FD_SCR_DATA_W(in)| FD_SCR_CPOL;
if(ss == FD_CS_PLL)
scr |= FD_SCR_SEL_PLL;
else if(ss == FD_CS_GPIO)
scr |= FD_SCR_SEL_GPIO;
fd_writel(scr, FD_REG_SCR);
fd_writel(scr | FD_SCR_START, FD_REG_SCR);
timeout = trtl_get_runtime_milliseconds() + 1000;
while (!(fd_readl(FD_REG_SCR) & FD_SCR_READY))
if (trtl_get_runtime_milliseconds() > timeout)
break;
if (!(fd_readl(FD_REG_SCR) & FD_SCR_READY))
return -EIO;
scr = fd_readl(FD_REG_SCR);
r = FD_SCR_DATA_R(scr);
if(out)
*out=r;
udelay(100); /* FIXME: check */
return 0;
}
static int fd_pll_writel(int val, int reg)
{
return fd_spi_xfer(FD_CS_PLL, 24, (reg << 8) | val, NULL);
}
static int fd_pll_readl(int reg)
{
uint32_t ret;
int err;
err = fd_spi_xfer(FD_CS_PLL, 24, (reg << 8) | (1 << 23), &ret);
if (err < 0)
return err;
return ret & 0xff;
}
int fd_pll_init(void)
{
int i, ret;
unsigned long timeout;
const struct ad9516_reg *r;
if (fd_pll_writel(0x99, 0x000) < 0)
goto out;
if (fd_pll_writel(0x01, 0x232) < 0)
goto out;
i = fd_pll_readl(0x003);
if (i < 0)
goto out;
if (i != 0xc3) {
pr_error("Error in PLL communication\n"
" (got 0x%x, expected 0xc3)\n", i);
return -EIO;
}
/* Write the magic config */
for (i = 0, r = __9516_regs; i < ARRAY_SIZE(__9516_regs); i++, r++) {
if (fd_pll_writel(r->val, r->reg) < 0) {
pr_error("Error in configuring PLL (step %i)\n", i);
return -EIO;
}
}
if (fd_pll_writel(0x01, 0x232) < 0)
goto out;
/* Wait for it to lock */
timeout = trtl_get_runtime_milliseconds() + 500;
do {
ret = fd_pll_readl(FD_REG_SCR);
if (ret < 0)
return -EIO;
if (ret & 0x1)
break;
} while (trtl_get_runtime_milliseconds() < timeout);
/*
* Synchronize the phase of all clock outputs
* (this is critical for the accuracy!)
*/
if (fd_pll_writel(0x01, 0x230) < 0)
goto out;
if (fd_pll_writel(0x01, 0x232) < 0)
goto out;
if (fd_pll_writel(0x00, 0x230) < 0)
goto out;
if (fd_pll_writel(0x01, 0x232) < 0)
goto out;
return 0;
out:
return -EIO;
}
software/firmware/fd/wrtd-fw-internals.h 0 → 100644
View file @ 704000a2
/*
* Copyright (C) 2018 CERN (www.cern.ch)
* Author: Federico Vaga <federico.vaga@cern.ch>
* Author: Tomasz Wlostowski <tomasz.wlostowski@cern.ch>
*
* SPDX-License-Identifier: LGPL-3.0-or-later
*/
extern int fd_init(void);
software/firmware/fd/wrtd-rt-fd.c
View file @ 704000a2
......@@ -16,6 +16,8 @@
#include "hw/fd_channel_regs.h"
#include "hw/fd_main_regs.h"
#include "fine-delay-init.h"
#include "loop-queue.h"
#define WRTD_OUT_HMQ 0
......@@ -950,6 +952,8 @@ static int wrtd_o_init(void)
rmq_bind_in(WRTD_OUT_RMQ, TRTL_EP_ETH, &addr);
fd_init();
pr_debug("rt-output firmware initialized.\n\r");
return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment