Date: 2022-09-28 - 2022-10-12
Raw results: charm3-sorted.log
Interesting facts:
- System board monimod was not reliable and required too many restart. So we decided to change the firmware and not scanning it anymore.
- So main cpu firmware was updated in operation. The spi flash was programmed, and the cpu was rebooted on the flash. The full process needs to be documented. This occurred on 2022-09-30 13:55.
Weird events:
- Cycle counter was reset for no reason:
2022-10-10 07:25:07,0,31,0,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,84,0xc953,864,0,2,0x13bc,209,0,32,49,3,00000f04,=sta=,0,46023,0,1821,80,80,4,2326227022,281251013,111021273,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,11,0x17a3,37,0,0,0x6242,41,0,3,0,=mbx0=,0,=plc1=,0,0,0,4,0x3c06,44,0,0,0xd93e,35,0,3,1,=mbx1=,1,0
2022-10-10 07:25:08,0,31,0,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,84,0xd877,864,0,2,0x76a1,209,0,32,49,3,00000f04,=sta=,0,46023,0,1821,80,80,4,15888,281251134,111021320,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,11,0x2e25,37,0,0,0xf3b1,41,0,3,0,=mbx0=,0,=plc1=,0,0,0,4,0x5124,44,0,0,0x6979,35,0,3,1,=mbx1=,1,0
plc cpu freezes:
- legend
time,ping,fip-err,comm-err,=sys=,cycle,P12V,P12A,P1.2V,P3.3V,Vperi,Tamb,Tldo,Tfpga,=fan=,FTP12V,FTP12A,FTP5V,FTtemp,fan1,fan2,fan3,adc4,adc5,adc6,adc7,Tfan2,=sv=,reset-cause,cpus-reset,iram-se-corr,iram-scrub-cyc,iram-scrub-se,iram-scrub-de,dram-se-corr,dram-scrub-cyc,dram-scrub-se,dram-scrub-de,insn-bus-err,data-bus-err,ram-scrub-cfg,iram-ecc-addr,=sta=,monimod-reset,mm-fantray-reset,radtopus-errs,min-watchdog,nbr-recovery,nbr-reset,last-reset,cycle-count,fip-cmds,fip-pad,build-date,hash,=plc0=,plc0-reset,plc0-status,plc0-reset-cause,plc0-iram-se-corr,plc0-iram-scrub-cyc,plc0-iram-scrub-se,plc0-iram-scrub-de,plc0-dram-se-corr,plc0-dram-scrub-cyc,plc0-dram-scrub-se,plc0-dram-scrub-de,plc0-scrub-cfg,plc0-cpu-fault,=mbx0=,mbx0-err,=plc1=,plc1-reset,plc1-status,plc1-reset-cause,plc1-iram-se-corr,plc1-iram-scrub-cyc,plc1-iram-scrub-se,plc1-iram-scrub-de,plc1-dram-se-corr,plc1-dram-scrub-cyc,plc1-dram-scrub-se,plc1-dram-scrub-de,plc1-scrub-cfg,plc1-cpu-fault,=mbx1=,mbx1-err,flash-err
- plc 0
2022-09-29 14:58:53,0,1,3,=sys=,0xff,11.748,0.197,1.235,3.346,2.488,20.781,22.469,23.438,=fan=,7.976,0.193,4.896,27.188,2010.000000,2010.000000,2010.000000,920,3162,3290,1182,27.00,=sv=,0x11,0x0,4,0x51b5,52,0,0,0x702a,13,0,2,6,3,00000bc4,=sta=,0,0,0,1820,8,8,1,3553371206,10411164,3728319,27-09-2022 17:18:58,d695dbd2,=plc0=,0,0,0,5,0x083c,7,0,0,0xbd1c,20,0,3,0,=mbx0=,0,=plc1=,0,0,0,3,0x3c6d,6,0,0,0x37b2,12,0,3,0,=mbx1=,1,0 ## Tfan2
2022-09-29 14:59:40,0,1,3,=sys=,0x26,11.746,0.198,1.235,3.346,2.488,20.781,22.438,23.406,=fan=,7.992,0.185,4.930,27.188,1980.000000,2010.000000,1980.000000,923,3163,3291,1195,26.00,=sv=,0x11,0x0,4,0xc6e0,52,0,0,0x486c,13,0,2,6,3,00000bc4,=sta=,0,0,0,1820,8,8,1,3561339467,10434602,3736706,27-09-2022 17:18:58,d695dbd2,=plc0=,0,0,11,5,0x1a0c,7,0,0,0xadf9,20,0,3,0,=mbx0=,0,=plc1=,0,0,0,3,0xa08f,6,0,0,0x42ac,12,0,3,0,=mbx1=,1,0 ## plc0-reset-cause
mailbox not ready
2022-09-29 14:59:40,0,1,3,=sys=,0x26,11.746,0.198,1.235,3.346,2.488,20.781,22.438,23.406,=fan=,7.992,0.185,4.930,27.188,1980.000000,2010.000000,1980.000000,923,3163,3291,1195,26.00,=sv=,0x11,0x0,4,0xd7e4,52,0,0,0xb796,13,0,2,6,3,00000bc4,=sta=,0,0,0,1820,8,8,1,3561385350,10434737,3736754,27-09-2022 17:18:58,d695dbd2,=plc0=,0,0,11,5,0x433f,7,0,0,0x2823,20,0,3,0,=mbx0=,1,=plc1=,0,0,0,3,0xb2f7,6,0,0,0xe622,12,0,3,0,=mbx1=,1,0 ## mbx0-err
reset-cause: recov+cpu div
- plc 0
2022-09-30 22:36:30,0,11,0,=sys=,0x2e,0,0,0,0,0,0,0,0,=fan=,11.254,0.447,4.745,25.781,0.000000,0.000000,1980.000000,915,3185,3285,1180,25.50,=sv=,0x11,0x0,5,0xefa2,48,0,0,0x148d,13,0,2,27,3,000003a4,=sta=,0,170,0,1820,29,29,2,1155944551,15836536,6011725,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,0,0x22e8,6,0,0,0x0abf,4,0,3,0,=mbx0=,0,=plc1=,0,0,0,0,0xf06a,7,0,0,0x8555,13,0,3,0,=mbx1=,0,0 ## plc1-iram-scrub-se
2022-09-30 22:41:24,0,11,0,=sys=,0x27,0,0,0,0,0,0,0,0,=fan=,11.300,0.438,4.761,25.562,0.000000,0.000000,2010.000000,920,3177,3284,1184,26.00,=sv=,0x11,0x0,5,0xbf3f,48,0,0,0x2d02,13,0,2,27,3,000003a4,=sta=,0,170,0,1820,29,29,2,1205888890,15983444,6066282,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0xe259,6,0,0,0xc6aa,4,0,3,0,=mbx0=,0,=plc1=,0,0,0,0,0xea7b,7,0,0,0x449e,13,0,3,0,=mbx1=,0,0 ## plc0-reset-cause
mailbox not ready
2022-09-30 22:41:24,0,11,0,=sys=,0x27,0,0,0,0,0,0,0,0,=fan=,11.300,0.438,4.761,25.562,0.000000,0.000000,2010.000000,920,3177,3284,1184,26.00,=sv=,0x11,0x0,5,0xcf03,48,0,0,0x93ff,13,0,2,27,3,000003a4,=sta=,0,170,0,1820,29,29,2,1205931729,15983570,6066329,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0x08cd,6,0,0,0x38b0,4,0,3,0,=mbx0=,1,=plc1=,0,0,0,0,0x007a,7,0,0,0xdadc,13,0,3,0,=mbx1=,0,0 ## mbx0-err
reset-cause: recov+cpu div
- plc 1
2022-10-01 08:50:59,0,11,2,=sys=,0x22,0,0,0,0,0,0,0,0,=fan=,63.999,0.192,4.935,27.812,1980.000000,1980.000000,1980.000000,944,3157,3290,1215,24.75,=sv=,0x11,0x0,16,0x708e,133,0,1,0x413d,39,0,8,33,3,000003e4,=sta=,0,14629,0,1820,41,41,1,3097283000,34270493,12964924,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0x3bfe,7,0,0,0xffa5,12,0,3,0,=mbx0=,0,=plc1=,0,0,0,5,0x6260,31,0,0,0x24b3,46,0,3,2,=mbx1=,2,0 ## mm-fantray-reset
2022-10-01 08:50:59,0,11,2,=sys=,0x22,0,0,0,0,0,0,0,0,=fan=,63.999,0.192,4.935,27.812,1980.000000,1980.000000,1980.000000,944,3157,3290,1215,24.75,=sv=,0x11,0x0,16,0x8fc3,133,0,1,0x1106,39,0,8,33,3,000003e4,=sta=,0,14629,0,1820,41,41,1,3097366885,34270745,12965018,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0x8897,7,0,0,0xe2c8,12,0,3,0,=mbx0=,0,=plc1=,0,1,1,5,0x8982,31,0,0,0x2b33,46,0,3,2,=mbx1=,2,0 ## plc1-status
mailbox not ready
2022-10-01 08:50:59,0,11,2,=sys=,0x22,0,0,0,0,0,0,0,0,=fan=,63.999,0.192,4.935,27.812,1980.000000,1980.000000,1980.000000,944,3157,3290,1215,24.75,=sv=,0x11,0x0,16,0x9f58,133,0,1,0x777e,39,0,8,33,3,000003e4,=sta=,0,14629,0,1820,41,41,1,3097409248,34270870,12965064,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0xaf0b,7,0,0,0x54ce,12,0,3,0,=mbx0=,0,=plc1=,0,1,1,5,0x8982,31,0,0,0x2b33,46,0,3,2,=mbx1=,3,0 ## mbx1-err
cpu in fault, reset-cause: cpu
- plc 1
2022-10-01 21:38:12,0,11,0,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,30,0xda5c,306,0,1,0x88d5,77,0,15,39,3,0000042c,=sta=,0,16608,0,1820,54,54,2,2535398083,57287158,21533302,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0x7a83,36,0,0,0x29b7,44,0,3,0,=mbx0=,0,=plc1=,0,0,0,6,0x2316,37,0,0,0xc45f,34,0,3,2,=mbx1=,1,0 ## plc0-dram-scrub-se
2022-10-01 21:39:15,0,11,0,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,30,0x9e07,306,0,1,0x1a1c,77,0,15,39,3,0000042c,=sta=,0,16608,0,1820,54,54,2,2546369013,57318323,21544875,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0xa16d,36,0,0,0x53f5,44,0,3,0,=mbx0=,0,=plc1=,0,1,1,6,0x8238,37,0,0,0xbb7c,34,0,3,2,=mbx1=,1,0 ## plc1-status
mailbox 1 not ready
2022-10-01 21:39:15,0,11,0,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,30,0xae36,306,0,1,0x8266,77,0,15,39,3,0000042c,=sta=,0,16608,0,1820,54,54,2,2546413016,57318448,21544921,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,0,0xc7e1,36,0,0,0xc5fa,44,0,3,0,=mbx0=,0,=plc1=,0,1,1,6,0x8238,37,0,0,0xbb7c,34,0,3,2,=mbx1=,2,0 ## mbx1-err
cpu in fault, reset-cause: cpu
- plc 1
2022-10-02 20:21:10,0,11,1,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,49,0x9169,523,0,2,0xb4a7,143,0,27,42,3,00000e8c,=sta=,0,16608,0,1820,69,69,4,4044322983,98175528,36755669,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,6,0x5f6d,30,0,0,0xd57a,22,0,3,0,=mbx0=,1,=plc1=,0,0,0,7,0x2dca,14,0,0,0x5814,25,0,3,0,=mbx1=,1,0 ## plc1-dram-scrub-se
mailbox 1 not ready
2022-10-02 20:22:08,0,11,1,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,49,0x304b,523,0,2,0xe78f,143,0,27,42,3,00000e8c,=sta=,0,16608,0,1820,69,69,4,4054497980,98204432,36766403,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,6,0x6000,30,0,0,0x8a43,22,0,3,0,=mbx0=,1,=plc1=,0,1,1,7,0xf5f3,14,0,0,0x7343,25,0,3,0,=mbx1=,2,0 ## plc1-status
mailbox 1 not ready
2022-10-02 20:22:08,0,11,1,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,49,0x409c,523,0,2,0x50ae,143,0,27,42,3,00000e8c,=sta=,0,16608,0,1820,69,69,4,4054542335,98204558,36766450,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,6,0x7770,30,0,0,0x21b5,22,0,3,0,=mbx0=,1,=plc1=,0,1,1,7,0xf5f3,14,0,0,0x7343,25,0,3,0,=mbx1=,3,0 ## mbx1-err
cpu in fault, reset-cause: cpu
- plc 0
2022-10-03 14:09:22,0,11,0,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,52,0x6262,588,0,2,0x7e1c,156,0,29,51,3,000002d4,=sta=,0,16608,0,1820,80,80,4,2440392312,130221252,48669456,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,3,0x8c85,9,0,0,0x139b,17,0,3,0,=mbx0=,1,=plc1=,0,0,0,1,0x276e,12,0,0,0x8e31,18,0,3,0,=mbx1=,1,0 ## plc0-iram-scrub-se
2022-10-03 14:14:49,0,11,0,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,52,0x26a2,588,0,2,0xbf4a,156,0,29,51,3,000002d4,=sta=,0,16608,0,1820,80,80,4,2497992428,130384876,48730222,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,3,0x7590,9,0,0,0x40c9,17,0,3,0,=mbx0=,1,=plc1=,0,0,0,1,0x9eac,12,0,0,0xc7ec,18,0,3,0,=mbx1=,1,0 ## plc0-reset-cause
mailbox 0 not ready
2022-10-03 14:14:50,0,11,0,=sys=,0x0e,0,0,0,0,0,0,0,0,=fan=,11.636,0.197,4.897,28.188,1980.000000,1980.000000,1980.000000,915,3182,3293,1179,26.00,=sv=,0x11,0x0,52,0x36d2,588,0,2,0x2795,156,0,29,51,3,000002d4,=sta=,0,16608,0,1820,80,80,4,2498036783,130385002,48730269,29-09-2022 08:38:50,8368f828,=plc0=,0,0,11,3,0x9c04,9,0,0,0xb2ce,17,0,3,0,=mbx0=,2,=plc1=,0,0,0,1,0xb4d8,12,0,0,0x5f5d,18,0,3,0,=mbx1=,1,0 ## mbx0-err
reset-cause: recov+div
- plc 1
2022-10-03 18:53:42,0,21,1,=sys=,0x00,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,0.000,0.000000,0.000000,0.000000,0,0,0,0,0.00,=sv=,0x11,0x0,3,0x7df0,14,0,0,0x978f,5,0,0,21,3,00000828,=sta=,0,0,1,1820,21,21,4,2740724477,7785206,3794185,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,0,0x8179,1,0,0,0x50b8,4,0,3,0,=mbx0=,1,=plc1=,0,1,1,0,0xddd6,2,0,0,0x7371,0,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox 1 not ready
2022-10-03 18:53:42,0,21,1,=sys=,0x00,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,0.000,0.000000,0.000000,0.000000,0,0,0,0,0.00,=sv=,0x11,0x0,3,0x8d98,14,0,0,0xff66,5,0,0,21,3,00000828,=sta=,0,0,1,1820,21,21,4,2740768833,7785332,3794232,29-09-2022 08:38:50,8368f828,=plc0=,0,0,0,0,0x98e9,1,0,0,0xe82a,4,0,3,0,=mbx0=,1,=plc1=,0,1,1,0,0xddd6,2,0,0,0x7371,0,0,3,0,=mbx1=,2,0 ## mbx1-err
cpu in fault, reset-cause: cpu
- plc 0
2022-10-05 17:10:09,0,31,0,=sys=,0x9d,0,0,0,0,0,0,0,0,=fan=,11.638,0.180,4.936,30.125,1980.000000,2010.000000,1980.000000,0,0,0,0,0,=sv=,0x11,0x0,18,0xb956,194,0,0,0x269a,44,0,5,6,3,00000f70,=sta=,0,4548,0,1821,11,11,4,2542497383,82803971,32356707,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,3,0x33f9,40,0,0,0x3148,37,0,3,0,=mbx0=,1,=plc1=,0,0,0,3,0x1d47,41,0,0,0xa70f,47,0,3,2,=mbx1=,1,0 ## dram-scrub-se
2022-10-05 17:14:33,0,31,0,=sys=,0x7d,0,0,0,0,0,0,0,0,=fan=,11.665,0.183,4.881,30.000,2010.000000,2010.000000,1980.000000,0,0,0,0,0,=sv=,0x11,0x0,18,0x308d,194,0,0,0x31e0,44,0,5,6,3,00000f70,=sta=,0,4548,0,1821,11,11,4,2587333993,82935840,32407694,03-10-2022 16:52:57,4e06d139,=plc0=,0,1,2,3,0xf6b8,40,0,0,0xeec5,37,0,3,0,=mbx0=,1,=plc1=,0,0,0,3,0xc0bf,41,0,0,0xc816,47,0,3,2,=mbx1=,1,0 ## plc0-status
mailbox not ready
2022-10-05 17:14:33,0,31,0,=sys=,0x7d,0,0,0,0,0,0,0,0,=fan=,11.665,0.183,4.881,30.000,2010.000000,2010.000000,1980.000000,0,0,0,0,0,=sv=,0x11,0x0,18,0x3f92,194,0,0,0x93f5,44,0,5,6,3,00000f70,=sta=,0,4548,0,1821,11,11,4,2587375136,82935961,32407741,03-10-2022 16:52:57,4e06d139,=plc0=,0,1,2,3,0xf6b8,40,0,0,0xeec5,37,0,3,0,=mbx0=,2,=plc1=,0,0,0,3,0xd60a,41,0,0,0x5986,47,0,3,2,=mbx1=,1,0 ## mbx0-err
status: 1 (fault), reset-cause: 2 (iram-ecc) ?
- plc 1
2022-10-06 00:31:13,0,31,2,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,24,0x1dbd,243,0,0,0xa972,58,0,6,12,3,00000d3c,=sta=,0,15749,0,1821,18,18,4,2722740783,96035997,37539575,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,5,0x29cf,11,0,0,0x9c8c,9,0,3,0,=mbx0=,1,=plc1=,0,0,0,3,0x3bc0,10,0,0,0x1855,19,0,3,0,=mbx1=,1,0 ## mm-fantray-reset
mailbox not ready
2022-10-06 00:31:14,0,31,2,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,24,0x2cbb,243,0,0,0x0d37,58,0,6,12,3,00000d3c,=sta=,0,15749,0,1821,18,18,4,2722781065,96036118,37539622,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,5,0x4050,11,0,0,0x2dfb,9,0,3,0,=mbx0=,1,=plc1=,0,1,1,3,0x3c7a,10,0,0,0x187f,19,0,3,0,=mbx1=,2,0 ## plc1-status
mailbox not ready
2022-10-06 00:31:14,0,31,2,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,24,0x3b97,243,0,0,0x7029,58,0,6,12,3,00000d3c,=sta=,0,15749,0,1821,18,18,4,2722821347,96036239,37539669,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,5,0x56d2,11,0,0,0xbf6b,9,0,3,0,=mbx0=,1,=plc1=,0,1,1,3,0x3c7a,10,0,0,0x187f,19,0,3,0,=mbx1=,3,0 ## mbx1-err
status: 1 (fault), reset-cause: 1 (cpu divergence)
- plc 0
2022-10-06 10:29:04,0,31,0,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,28,0x6d70,329,0,1,0x8994,85,0,8,19,3,00000c50,=sta=,0,30045,0,1821,27,27,1,263850844,113971134,46203470,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,2,0x2b85,4,0,0,0x5361,5,0,3,0,=mbx0=,1,=plc1=,0,0,0,0,0xd5a6,6,0,0,0xc7f4,7,0,3,0,=mbx1=,1,0 ## dram-scrub-se
2022-10-06 10:33:30,0,31,0,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,28,0xbfaf,329,0,1,0x3fcc,85,0,8,19,3,00000c50,=sta=,0,30045,0,1821,27,27,1,310701771,114104210,46254924,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,11,2,0xe0d9,4,0,0,0x1571,5,0,3,0,=mbx0=,1,=plc1=,0,0,0,0,0x4d7f,6,0,0,0x93b4,7,0,3,0,=mbx1=,1,0 ## plc0-reset-cause
mailbox not ready
2022-10-06 10:33:30,0,31,0,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,28,0xcf5b,329,0,1,0xa4c1,85,0,8,19,3,00000c50,=sta=,0,30045,0,1821,27,27,1,310744370,114104331,46254971,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,11,2,0x05c7,4,0,0,0x82f0,5,0,3,0,=mbx0=,2,=plc1=,0,0,0,0,0x62ca,6,0,0,0x2523,7,0,3,0,=mbx1=,1,0 ## mbx0-err
status: 0, reset-cause: 11 (cpu divergence + recovery)
- plc 1
2022-10-06 12:44:04,0,31,0,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,31,0x3b12,348,0,1,0x1c80,88,0,8,25,3,000000b4,=sta=,0,30975,0,1821,33,33,4,1677791370,118021168,47845557,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,1,0x79ba,5,0,0,0x4148,7,0,3,0,=mbx0=,1,=plc1=,0,0,0,1,0x988f,10,0,0,0xf413,8,0,3,0,=mbx1=,1,0 ## mm-fantray-reset
2022-10-06 12:44:05,0,31,0,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,31,0x7721,348,0,1,0xa7df,88,0,8,25,3,000000b4,=sta=,0,30975,0,1821,33,33,4,1677954024,118021650,47845743,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,1,0xd363,5,0,0,0x849d,7,0,3,0,=mbx0=,1,=plc1=,0,1,2,1,0xdbf4,10,0,0,0xbb9f,8,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-06 12:44:05,0,31,0,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,31,0x8645,348,0,1,0x0ac5,88,0,8,25,3,000000b4,=sta=,0,30975,0,1821,33,33,4,1677995167,118021771,47845790,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,1,0xe9e5,5,0,0,0x160c,7,0,3,0,=mbx0=,1,=plc1=,0,1,2,1,0xdbf4,10,0,0,0xbb9f,8,0,3,0,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 2 (iram-ecc)
- plc 1
2022-10-07 17:58:38,0,31,4,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,37,0x848f,426,0,1,0xa9f1,101,0,12,34,3,00000f7c,=sta=,0,36669,0,1821,46,46,2,2956028460,170657410,68227295,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,2,0x4a3c,10,0,0,0xcff3,16,0,3,0,=mbx0=,1,=plc1=,0,0,0,1,0x5a6f,12,0,0,0x4486,19,0,3,0,=mbx1=,1,0 ## dram-scrub-se
2022-10-07 17:59:35,0,31,4,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,37,0x7388,426,0,1,0x1322,101,0,12,34,3,00000f7c,=sta=,0,36669,0,1821,46,46,2,2966136143,170686120,68238393,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,2,0x26b8,10,0,0,0x9b8e,16,0,3,0,=mbx0=,1,=plc1=,0,1,1,1,0x11e0,12,0,0,0xed83,19,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-07 17:59:35,0,31,4,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,37,0x8290,426,0,1,0x76d8,101,0,12,34,3,00000f7c,=sta=,0,36669,0,1821,46,46,2,2966178390,170686240,68238439,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,2,0x3d0b,10,0,0,0x2bca,16,0,3,0,=mbx0=,1,=plc1=,0,1,1,1,0x11e0,12,0,0,0xed83,19,0,3,0,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 1 (cpu divergence)
- plc 1
2022-10-08 13:27:34,0,31,3,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,63,0x151d,675,0,1,0x5b74,162,0,18,38,3,00000750,=sta=,0,36669,0,1821,56,56,2,2417109881,205725102,81799301,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,16,0x1ccf,57,0,0,0x4fe1,72,0,3,0,=mbx0=,0,=plc1=,0,0,0,8,0x5ce3,53,0,0,0xcbaa,47,0,3,0,=mbx1=,1,0 ## plc0-dram-scrub-se
2022-10-08 13:29:11,0,31,3,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,63,0xae31,675,0,1,0xdd9c,162,0,18,38,3,00000750,=sta=,0,36669,0,1821,56,56,2,2434227813,205773724,81818101,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,16,0x7139,57,0,0,0x9875,72,0,3,0,=mbx0=,0,=plc1=,0,1,1,8,0xbb11,53,0,0,0xb453,47,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-08 13:29:11,0,31,3,=sys=,0x58,0,0,0,0,0,0,0,0,=fan=,11.513,0.186,4.874,29.938,2010.000000,1980.000000,2010.000000,0,0,0,0,0,=sv=,0x11,0x0,63,0xbd1a,675,0,1,0x407e,162,0,18,38,3,00000750,=sta=,0,36669,0,1821,56,56,2,2434270413,205773845,81818148,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,16,0x87bb,57,0,0,0x29e4,72,0,3,0,=mbx0=,0,=plc1=,0,1,1,8,0xbb11,53,0,0,0xb453,47,0,3,0,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 1 (cpu divergence)
- plc 1
2022-10-08 14:39:45,0,31,0,=sys=,0x11,0,0,0,0,0,0,0,0,=fan=,12.237,0.197,5.255,26.312,4224.000000,2128.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,65,0xf890,697,0,1,0xbc84,168,0,20,41,3,00000cb0,=sta=,0,36669,0,1821,61,61,4,3172673425,207890443,82650985,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,0,0xe8bf,1,0,0,0x7f85,2,0,3,0,=mbx0=,0,=plc1=,0,0,0,0,0x2996,1,0,0,0xfb4f,2,0,3,0,=mbx1=,1,0 ## FTP12V
2022-10-08 14:39:45,0,31,0,=sys=,0x11,0,0,0,0,0,0,0,0,=fan=,12.237,0.197,5.255,26.312,4224.000000,2128.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,65,0x07b5,697,0,1,0x1f6a,168,0,20,41,3,00000cb0,=sta=,0,36669,0,1821,61,61,4,3172714568,207890564,82651032,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,0,0xff41,1,0,0,0x10f5,2,0,3,0,=mbx0=,0,=plc1=,0,1,1,0,0x320e,1,0,0,0x3055,2,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-08 14:39:45,0,31,0,=sys=,0x11,0,0,0,0,0,0,0,0,=fan=,12.237,0.197,5.255,26.312,4224.000000,2128.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,65,0x16b9,697,0,1,0x817f,168,0,20,41,3,00000cb0,=sta=,0,36669,0,1821,61,61,4,3172755711,207890685,82651079,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,0,0x15c3,1,0,0,0xa263,2,0,3,0,=mbx0=,0,=plc1=,0,1,1,0,0x320e,1,0,0,0x3055,2,0,3,0,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 1 (cpu divergence)
- plc 1
2022-10-09 15:44:03,0,31,0,=sys=,0x17,0,0,0,0,0,0,0,0,=fan=,13.093,0.159,5.630,24.469,2068.000000,15888.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,65,0xbc3c,703,0,1,0x638e,170,0,21,44,3,00000184,=sta=,0,36669,0,1821,65,65,1,1337002088,253019266,100102262,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,1,0x3794,0,0,0,0x3e29,1,0,3,0,=mbx0=,0,=plc1=,0,0,0,0,0x80c7,0,0,0,0xb3f0,0,0,3,0,=mbx1=,1,0 ## FTP5V
2022-10-09 15:44:04,0,31,0,=sys=,0x17,0,0,0,0,0,0,0,0,=fan=,13.093,0.159,5.630,24.469,2068.000000,15888.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,65,0xe989,703,0,1,0x8b6f,170,0,21,44,3,00000184,=sta=,0,36669,0,1821,65,65,1,1337125176,253019628,100102402,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,1,0x7b1a,0,0,0,0xf276,1,0,3,0,=mbx0=,0,=plc1=,0,1,1,0,0xbea4,0,0,0,0x55b4,0,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-09 15:44:04,0,31,0,=sys=,0x17,0,0,0,0,0,0,0,0,=fan=,13.093,0.159,5.630,24.469,2068.000000,15888.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,65,0xf8ae,703,0,1,0xee5a,170,0,21,44,3,00000184,=sta=,0,36669,0,1821,65,65,1,1337166310,253019749,100102449,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,1,0x916d,0,0,0,0x82b2,1,0,3,0,=mbx0=,0,=plc1=,0,1,1,0,0xbea4,0,0,0,0x55b4,0,0,3,0,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 1 (cpu divergence)
- plc 1
2022-10-10 10:04:24,0,31,1,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,87,0x857b,891,0,3,0xbfd8,217,0,32,49,3,00000c28,=sta=,0,51401,0,1821,80,80,4,1613253931,286029442,112868913,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,16,0x3236,42,0,0,0x6c4e,49,0,3,0,=mbx0=,0,=plc1=,0,0,0,5,0xa73a,55,0,0,0xe217,42,0,3,1,=mbx1=,1,0 ## mm-fantray-reset
2022-10-10 10:04:26,0,31,1,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,87,0xd0a9,891,0,3,0xae3a,217,0,32,49,3,00000c28,=sta=,0,51401,0,1821,80,80,4,1613457611,286030045,112869146,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,16,0xa260,42,0,0,0x4113,49,0,3,0,=mbx0=,0,=plc1=,0,1,1,5,0x089f,55,0,0,0x768c,42,0,3,1,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-10 10:04:26,0,31,1,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,87,0xdfcd,891,0,3,0x1120,217,0,32,49,3,00000c28,=sta=,0,51401,0,1821,80,80,4,1613498754,286030166,112869193,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,16,0xb8e2,42,0,0,0xd283,49,0,3,0,=mbx0=,0,=plc1=,0,1,1,5,0x089f,55,0,0,0x768c,42,0,3,1,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 1 (cpu divergence)
- plc 1
2022-10-10 22:34:09,0,31,1,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,101,0xe53a,1015,0,3,0xf267,242,0,38,55,3,000001fc,=sta=,0,52766,0,1821,92,92,1,924394488,308521545,121578862,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,8,0x3156,23,0,0,0x8964,37,0,3,0,=mbx0=,1,=plc1=,0,0,0,5,0x3347,28,0,0,0x03fa,30,0,3,1,=mbx1=,1,0 ## iram-scrub-se
2022-10-10 22:35:06,0,31,1,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,101,0xc668,1015,0,3,0x001c,242,0,38,55,3,000001fc,=sta=,0,52766,0,1821,92,92,1,934462740,308550143,121589922,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,8,0xf8fd,23,0,0,0xce60,37,0,3,0,=mbx0=,1,=plc1=,0,1,8,5,0xd5d9,28,0,0,0x1e38,30,0,3,1,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-10 22:35:06,0,31,1,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,101,0xd551,1015,0,3,0x62fe,242,0,38,55,3,000001fc,=sta=,0,52766,0,1821,92,92,1,934505339,308550264,121589969,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,8,0x0f7f,23,0,0,0x5fd0,37,0,3,0,=mbx0=,1,=plc1=,0,1,8,5,0xd5d9,28,0,0,0x1e38,30,0,3,1,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 8 (watchdog)
- plc 1
2022-10-11 07:51:27,0,31,0,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,109,0x1a15,1107,0,3,0xb85a,259,0,39,59,3,00000bf8,=sta=,0,52766,0,1821,96,96,4,2515466990,325240332,128046376,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,0,0x28aa,4,0,0,0x4f43,4,0,3,0,=mbx0=,0,=plc1=,0,0,0,1,0xaac9,2,0,0,0xc9d8,5,0,3,0,=mbx1=,1,0 ## iram-scrub-se
2022-10-11 07:53:51,0,31,0,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,109,0xffd8,1107,0,3,0x2ba6,259,0,39,59,3,00000bf8,=sta=,0,52766,0,1821,96,96,4,2540781612,325312236,128074176,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,0,0x67df,4,0,0,0xe752,4,0,3,0,=mbx0=,0,=plc1=,0,1,1,1,0x0b0a,2,0,0,0x0b66,5,0,3,0,=mbx1=,1,0 ## plc1-status
mailbox not ready
2022-10-11 07:53:51,0,31,0,=sys=,0x14,0,0,0,0,0,0,0,0,=fan=,0.000,0.000,0.000,23.938,2188.000000,0.000000,0.000000,0,0,0,0,0,=sv=,0x11,0x0,109,0x0ee0,1107,0,3,0x8f5c,259,0,39,59,3,00000bf8,=sta=,0,52766,0,1821,96,96,4,2540824211,325312357,128074223,03-10-2022 16:52:57,4e06d139,=plc0=,0,0,0,0,0x7e61,4,0,0,0x78c2,4,0,3,0,=mbx0=,0,=plc1=,0,1,1,1,0x0b0a,2,0,0,0x0b66,5,0,3,0,=mbx1=,2,0 ## mbx1-err
status: 1, reset-cause: 1 (cpu divergence)
To check: plc-0: cpu recovery
TODO
- Add millisecond to timestamp (at least 10th of sec)
- Remove version and time in log
- i2c poweroff
- per i2c bus reset