Implement frame synchronization

There is an issue somewhere between the mod and demod, since frame
synchronization works on a direct path.

6 files changed, 169 insertions, 103 deletions

diff --git a/src/gr-fadingui/python/CMakeLists.txt b/src/gr-fadingui/python/CMakeLists.txt
index 1318857..6ee73d3 100644
--- a/src/gr-fadingui/python/CMakeLists.txt
+++ b/src/gr-fadingui/python/CMakeLists.txt
@@ -47,4 +47,4 @@ include(GrTest)
 
 set(GR_TEST_TARGET_DEPS gnuradio-fadingui)
 set(GR_TEST_PYTHON_DIRS ${CMAKE_BINARY_DIR}/swig)
-GR_ADD_TEST(qa_dearpygui_sink ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/qa_dearpygui_sink.py)
+GR_ADD_TEST(qa_xor_frame_sync ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/qa_xor_frame_sync.py)
diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py
index 12573a8..38771c5 100644
--- a/src/gr-fadingui/python/datasource.py
+++ b/src/gr-fadingui/python/datasource.py
@@ -35,7 +35,7 @@ class datasource(gr.basic_block):
         # a frame has 5 id bits so, there can only be 2 ** 5 chunks per file
         # see docstring of frame_obj for more details
         nblocks = int(self.fsize / self.frame.payload_length)
-        log.debug(f"Loaded {self.fsize} bytes = {nblocks} blocks from {self.fname}")
+        log.debug(f"Loaded {self.fsize} bytes == {nblocks} blocks from {self.fname}")
         assert nblocks < 2 ** 5, "Payload size too small or file too big"
 
         self.fpos = 0
@@ -73,7 +73,7 @@ class datasource(gr.basic_block):
         self.outbuffer = frame_bytes[len(out):]
 
         log.debug(f"Sent frame nr={self.blocknr}")
-        log.debug(f"Set bytes {out}")
+        log.debug(f"Sent bytes {out}")
 
         self.fpos += self.frame.payload_length
         self.blocknr += 1
diff --git a/src/gr-fadingui/python/qa_xor_frame_sync.py b/src/gr-fadingui/python/qa_xor_frame_sync.py
new file mode 100644
index 0000000..280c694
--- /dev/null
+++ b/src/gr-fadingui/python/qa_xor_frame_sync.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+from gnuradio import gr, gr_unittest, blocks
+
+from xor_frame_sync import xor_frame_sync
+import numpy as np
+
+
+class test_xor_frame_sync(gr_unittest.TestCase):
+
+    def setUp(self):
+        self.tb = gr.top_block()
+
+    def tearDown(self):
+        self.tb = None
+
+    def test_001(self):
+        """Test a byte aligned delay"""
+        pattern = np.array([0xbe, 0xef], dtype=np.uint8)
+        testdata = np.packbits(np.concatenate([
+            np.unpackbits(np.arange(0, 10, dtype=np.uint8)),
+            np.random.randint(0, 2, size = 8 * 5), np.unpackbits(pattern),
+            np.random.randint(0, 2, size = 64)
+        ]))
+
+        src = blocks.vector_source_b(testdata)
+        op = xor_frame_sync(pattern, 2048)
+        dst = blocks.vector_sink_b()
+
+        self.tb.connect(src, op, dst)
+        self.tb.run()
+
+        self.assertEqual(op.synchronized, True)
+
+    # FIXME: implement feature
+    # def test_002(self):
+    #     """Test a byte unaligned delay"""
+    #     pattern = np.array([0xbe, 0xef], dtype=np.uint8)
+    #     testdata = np.packbits(np.concatenate([
+    #         np.unpackbits(np.arange(0, 10, dtype=np.uint8)),
+    #         np.random.randint(0, 2, size = (2 + 8 * 5)), np.unpackbits(pattern),
+    #         np.random.randint(0, 2, size = 64)
+    #     ]))
+
+    #     src = blocks.vector_source_b(testdata)
+    #     op = xor_frame_sync(pattern, 2048)
+    #     dst = blocks.vector_sink_b()
+
+    #     self.tb.connect(src, op, dst)
+    #     self.tb.run()
+
+    #     self.assertEqual(op.synchronized, True)
+
+
+if __name__ == "__main__":
+    gr_unittest.run(test_xor_frame_sync)
diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py
index 51af35a..e52b493 100644
--- a/src/gr-fadingui/python/xor_frame_sync.py
+++ b/src/gr-fadingui/python/xor_frame_sync.py
@@ -25,23 +25,24 @@ class xor_frame_sync(gr.sync_block):
             out_sig=[np.byte])
 
         # binary pattern to match
-        self.pattern = np.unpackbits(np.array(sync_pattern, dtype=np.uint8))[::-1]
-        self.nbytes = len(sync_pattern)
-        self.nbits = len(self.pattern)
+        self.pattern = sync_pattern
+        self.nbytes = len(self.pattern)
 
-        log.debug(f"Loaded pattern {self.pattern} length={self.nbits}")
+        self.pattern_bits = np.unpackbits(np.array(self.pattern, dtype=np.uint8))[::-1]
+        self.nbits = len(self.pattern_bits)
+
+        log.debug(f"Loaded pattern {self.pattern_bits} length={self.nbits}")
         assert(self.nbits % 8 == 0)
 
         # packed buffer to delay the data
         self.delaybuf = RingBuffer(buffer_size, dtype=np.uint8)
         self.delay = 0
 
-        # unpacked buffer to compute correlation values, initially filled with zeros
-        self.corrbuf = RingBuffer(self.nbits)
-        self.corrbuf.extend(np.zeros(self.nbits))
+        log.debug(f"Created delay ring buffer of size {self.delaybuf.maxlen}")
 
-        # buffer to store correlation values
-        self.xcorrs = RingBuffer(buffer_size)
+        # unpacked buffer to compute correlation values, initially filled with zeros
+        self.corrbuf = RingBuffer(self.nbits, dtype=np.uint8)
+        self.corrbuf.extend(np.zeros(self.corrbuf.maxlen))
 
         # synchronization state
         self.synchronized = False
@@ -53,11 +54,21 @@ class xor_frame_sync(gr.sync_block):
 
         Binary correlation between two bit vectors is just size of the
         vector(s) minus the number of bits that differ.
+
+        @return: Number of bits of v that were shifted into the buffer
+                 when the correlation matched. If no match is found
+                 the return value is None.
         """
-        v_arr = np.array(v, dtype=np.uint8)
-        for b in np.unpackbits(v_arr):
+        # this could be much faster with shifts, bitwise or and xor
+        # but this should do alright for the moment
+        v_bits = np.unpackbits(np.array(v, dtype=np.uint8))
+        for bitnr, b in enumerate(v_bits):
             self.corrbuf.appendleft(b)
-            yield self.nbits - np.sum(np.logical_xor(self.corrbuf, self.pattern))
+            if (np.bitwise_xor(self.corrbuf, self.pattern_bits) == 0).all():
+                return bitnr
+
+        # no cross correlation found
+        return None
 
     def work(self, input_items, output_items):
         """
@@ -68,35 +79,59 @@ class xor_frame_sync(gr.sync_block):
 
             - If the buffer is not synchronized, compute a binary cross
               correlation to find how much the stream should be delayed.
-
-        Notes:
-
-            - Even though the block input is of type np.byte, inp is an array
-              of 255 bytes, probably for performance reasons.
-              TODO: block processing
         """
         inp = input_items[0]
         out = output_items[0]
 
-        if not self.synchronized:
-            for v in inp:
-                # compute the cross correlation
-                xcs = self.xcorrelation(v)
+        inp_len = len(inp)
 
-                # add cross correlations to buffer and save value
-                self.xcorrs.extend(list(xcs))
-                self.delaybuf.appendleft(v)
+        if not self.synchronized:
+            if inp_len > self.delaybuf.maxlen:
+                log.error("Input is bigger than delay buffer")
 
-            peak = np.argmax(self.xcorrs)
-            if self.xcorrs[peak] == self.nbits:
-                self.delay = peak
-                self.synchronized = True
-                log.debug(f"Synchronized with delay={peak}")
+                # FIXME: Makes the QA hang for some reason
+                # raise NotImplemented
 
-            else:
-                self.synchronized = False
-                log.warning(f"Did not find a peak (max={self.xcorrs[peak]}, should be {self.nbits})")
+            for bytenr, value in enumerate(inp):
+                # save value in the buffer
+                self.delaybuf.appendleft(value)
 
+                # compute the cross correlation
+                bitnr = self.xcorrelation(value)
+                if bitnr is not None:
+                    # correlation was found
+                    delay_bits = 8 * bytenr + bitnr
+
+                    # FIXME: add bit delay
+                    self.delay = bytenr
+                    self.synchronized = True
+                    log.debug(f"Synchronized with delay={self.delay} delay_bits={delay_bits}")
+
+                    # Not aligned to bytes
+                    if bitnr != 7:
+                        log.error("Not implemented: byte unaligned delay")
+                        self.synchronized = False
+                        self.delay = 0
+
+                        # FIXME: Makes the QA hang for some reason
+                        # raise NotImplemented
+
+                    # bigger than buffer
+                    if bytenr > self.delaybuf.maxlen:
+                        log.error("Too too long to synchronize, ran out of buffer memory")
+                        self.synchronized = False
+                        self.delay = 0
+
+                        # FIXME: Makes the QA hang for some reason
+                        # raise NotImplemented
+
+                    # stop processing inputs
+                    break
+
+            if not self.synchronized:
+                log.warning(f"Processed {inp_len} samples but could not synchronize")
+        else:
+            self.delaybuf.extendleft(inp)
 
         # return data with delay
         out[:] = self.delaybuf[self.delay]
diff --git a/tests/fadingui/QAM/qam_nogui.grc b/tests/fadingui/QAM/qam_nogui.grc
index c3d886e..78dc691 100644
--- a/tests/fadingui/QAM/qam_nogui.grc
+++ b/tests/fadingui/QAM/qam_nogui.grc
@@ -66,7 +66,7 @@ blocks:
   id: variable
   parameters:
     comment: ''
-    value: 10e-3
+    value: '.002'
   states:
     bus_sink: false
     bus_source: false
@@ -164,7 +164,7 @@ blocks:
   id: variable
   parameters:
     comment: ''
-    value: '.002'
+    value: '.01'
   states:
     bus_sink: false
     bus_source: false
@@ -246,7 +246,7 @@ blocks:
     bus_sink: false
     bus_source: false
     bus_structure: null
-    coordinate: [2824, 312.0]
+    coordinate: [3056, 280.0]
     rotation: 0
     state: true
 - name: blocks_throttle_0
@@ -268,6 +268,25 @@ blocks:
     coordinate: [720, 356.0]
     rotation: 0
     state: enabled
+- name: blocks_throttle_1
+  id: blocks_throttle
+  parameters:
+    affinity: ''
+    alias: ''
+    comment: ''
+    ignoretag: 'True'
+    maxoutbuf: '0'
+    minoutbuf: '0'
+    samples_per_second: samp_rate
+    type: byte
+    vlen: '1'
+  states:
+    bus_sink: false
+    bus_source: false
+    bus_structure: null
+    coordinate: [1448, 772.0]
+    rotation: 0
+    state: disabled
 - name: channels_channel_model_0
   id: channels_channel_model
   parameters:
@@ -307,7 +326,7 @@ blocks:
     bus_structure: null
     coordinate: [1504, 284.0]
     rotation: 0
-    state: true
+    state: enabled
 - name: digital_constellation_decoder_cb_0
   id: digital_constellation_decoder_cb
   parameters:
@@ -323,7 +342,7 @@ blocks:
     bus_structure: null
     coordinate: [2152, 276.0]
     rotation: 0
-    state: true
+    state: enabled
 - name: digital_constellation_modulator_0
   id: digital_constellation_modulator
   parameters:
@@ -362,7 +381,7 @@ blocks:
     bus_structure: null
     coordinate: [1792, 280.0]
     rotation: 0
-    state: true
+    state: enabled
 - name: digital_diff_decoder_bb_0
   id: digital_diff_decoder_bb
   parameters:
@@ -376,9 +395,9 @@ blocks:
     bus_sink: false
     bus_source: false
     bus_structure: null
-    coordinate: [2192, 348.0]
-    rotation: 180
-    state: true
+    coordinate: [2408, 276.0]
+    rotation: 0
+    state: enabled
 - name: digital_map_bb_0
   id: digital_map_bb
   parameters:
@@ -392,9 +411,9 @@ blocks:
     bus_sink: false
     bus_source: false
     bus_structure: null
-    coordinate: [2144, 452.0]
+    coordinate: [2304, 372.0]
     rotation: 0
-    state: true
+    state: disabled
 - name: digital_pfb_clock_sync_xxx_0
   id: digital_pfb_clock_sync_xxx
   parameters:
@@ -417,7 +436,7 @@ blocks:
     bus_structure: null
     coordinate: [1184, 308.0]
     rotation: 0
-    state: true
+    state: enabled
 - name: fadingui_datasource_0
   id: fadingui_datasource
   parameters:
@@ -463,7 +482,7 @@ blocks:
     bus_sink: false
     bus_source: false
     bus_structure: null
-    coordinate: [2592, 452.0]
+    coordinate: [2728, 276.0]
     rotation: 0
     state: true
 - name: fadingui_xor_frame_sync_0
@@ -471,7 +490,7 @@ blocks:
   parameters:
     affinity: ''
     alias: ''
-    buffer_size: frame.length * 4
+    buffer_size: frame.length * 5
     comment: ''
     maxoutbuf: '0'
     minoutbuf: '0'
@@ -480,7 +499,7 @@ blocks:
     bus_sink: false
     bus_source: false
     bus_structure: null
-    coordinate: [2312, 444.0]
+    coordinate: [2512, 364.0]
     rotation: 0
     state: true
 - name: qtgui_const_sink_x_0
@@ -578,6 +597,7 @@ blocks:
 
 connections:
 - [blocks_throttle_0, '0', channels_channel_model_0, '0']
+- [blocks_throttle_1, '0', fadingui_xor_frame_sync_0, '0']
 - [channels_channel_model_0, '0', digital_pfb_clock_sync_xxx_0, '0']
 - [digital_cma_equalizer_cc_0, '0', digital_costas_loop_cc_0, '0']
 - [digital_cma_equalizer_cc_0, '0', qtgui_const_sink_x_0, '1']
@@ -587,9 +607,11 @@ connections:
 - [digital_costas_loop_cc_0, '0', fadingui_dearpygui_sink_0, '0']
 - [digital_costas_loop_cc_0, '0', qtgui_const_sink_x_0, '2']
 - [digital_diff_decoder_bb_0, '0', digital_map_bb_0, '0']
+- [digital_diff_decoder_bb_0, '0', fadingui_xor_frame_sync_0, '0']
 - [digital_map_bb_0, '0', fadingui_xor_frame_sync_0, '0']
 - [digital_pfb_clock_sync_xxx_0, '0', digital_cma_equalizer_cc_0, '0']
 - [digital_pfb_clock_sync_xxx_0, '0', qtgui_const_sink_x_0, '0']
+- [fadingui_datasource_0, '0', blocks_throttle_1, '0']
 - [fadingui_datasource_0, '0', digital_constellation_modulator_0, '0']
 - [fadingui_deframer_0, '0', blocks_null_sink_0, '0']
 - [fadingui_xor_frame_sync_0, '0', fadingui_deframer_0, '0']
diff --git a/tests/fadingui/QAM/qam_nogui.py b/tests/fadingui/QAM/qam_nogui.py
index 6cde63d..4116db5 100755
--- a/tests/fadingui/QAM/qam_nogui.py
+++ b/tests/fadingui/QAM/qam_nogui.py
@@ -21,9 +21,6 @@ if __name__ == '__main__':
         except:
             print("Warning: failed to XInitThreads()")
 
-from PyQt5 import Qt
-from gnuradio import qtgui
-import sip
 from gnuradio import blocks
 from gnuradio import channels
 from gnuradio.filter import firdes
@@ -31,6 +28,7 @@ from gnuradio import digital
 from gnuradio import gr
 import sys
 import signal
+from PyQt5 import Qt
 from argparse import ArgumentParser
 from gnuradio.eng_arg import eng_float, intx
 from gnuradio import eng_notation
@@ -81,64 +79,23 @@ class qam_nogui(gr.top_block, Qt.QWidget):
         self.time_offset = time_offset = 1
         self.samp_rate = samp_rate = 32000
         self.rrc_taps = rrc_taps = firdes.root_raised_cosine(nfilts, nfilts, 1.0/float(sps), excess_bw, 45*nfilts)
-        self.phase_bw = phase_bw = .002
+        self.phase_bw = phase_bw = .01
         self.noise_volt = noise_volt = 0
         self.freq_offset = freq_offset = 0
         self.frame = frame = fadingui.frame_obj(preamble=[190, 239], payload_len=32768)
         self.eq_ntaps = eq_ntaps = 15
         self.eq_mod = eq_mod = 1
-        self.eq_gain = eq_gain = 10e-3
+        self.eq_gain = eq_gain = .002
         self.const = const = digital.constellation_16qam().base()
         self.chn_taps = chn_taps = [1.0 + 0.0j, ]
 
         ##################################################
         # Blocks
         ##################################################
-        self.qtgui_const_sink_x_0 = qtgui.const_sink_c(
-            1024, #size
-            "", #name
-            3 #number of inputs
-        )
-        self.qtgui_const_sink_x_0.set_update_time(0.10)
-        self.qtgui_const_sink_x_0.set_y_axis(-2, 2)
-        self.qtgui_const_sink_x_0.set_x_axis(-2, 2)
-        self.qtgui_const_sink_x_0.set_trigger_mode(qtgui.TRIG_MODE_FREE, qtgui.TRIG_SLOPE_POS, 0.0, 0, "")
-        self.qtgui_const_sink_x_0.enable_autoscale(False)
-        self.qtgui_const_sink_x_0.enable_grid(False)
-        self.qtgui_const_sink_x_0.enable_axis_labels(True)
-
-
-        labels = ['', '', '', '', '',
-            '', '', '', '', '']
-        widths = [1, 1, 1, 1, 1,
-            1, 1, 1, 1, 1]
-        colors = ["blue", "red", "red", "red", "red",
-            "red", "red", "red", "red", "red"]
-        styles = [0, 0, 0, 0, 0,
-            0, 0, 0, 0, 0]
-        markers = [0, 0, 0, 0, 0,
-            0, 0, 0, 0, 0]
-        alphas = [1.0, 1.0, 1.0, 1.0, 1.0,
-            1.0, 1.0, 1.0, 1.0, 1.0]
-
-        for i in range(3):
-            if len(labels[i]) == 0:
-                self.qtgui_const_sink_x_0.set_line_label(i, "Data {0}".format(i))
-            else:
-                self.qtgui_const_sink_x_0.set_line_label(i, labels[i])
-            self.qtgui_const_sink_x_0.set_line_width(i, widths[i])
-            self.qtgui_const_sink_x_0.set_line_color(i, colors[i])
-            self.qtgui_const_sink_x_0.set_line_style(i, styles[i])
-            self.qtgui_const_sink_x_0.set_line_marker(i, markers[i])
-            self.qtgui_const_sink_x_0.set_line_alpha(i, alphas[i])
-
-        self._qtgui_const_sink_x_0_win = sip.wrapinstance(self.qtgui_const_sink_x_0.pyqwidget(), Qt.QWidget)
-        self.top_grid_layout.addWidget(self._qtgui_const_sink_x_0_win)
-        self.fadingui_xor_frame_sync_0 = fadingui.xor_frame_sync(sync_pattern=frame.preamble, buffer_size=frame.length * 4)
+        self.fadingui_xor_frame_sync_0 = fadingui.xor_frame_sync(sync_pattern=frame.preamble, buffer_size=frame.length * 5)
         self.fadingui_deframer_0 = fadingui.deframer(frame_obj=frame)
         self.fadingui_datasource_0 = fadingui.datasource(frame_obj=frame, filename='/home/god/Documents/Fading/tests/fadingui/QAM/lena512color.tiff')
         self.digital_pfb_clock_sync_xxx_0 = digital.pfb_clock_sync_ccf(sps, timing_loop_bw, rrc_taps, nfilts, nfilts/2, 1.5, 1)
-        self.digital_map_bb_0 = digital.map_bb([0, 1, 3, 2])
         self.digital_diff_decoder_bb_0 = digital.diff_decoder_bb(4)
         self.digital_costas_loop_cc_0 = digital.costas_loop_cc(phase_bw, 4, False)
         self.digital_constellation_modulator_0 = digital.generic_mod(
@@ -169,15 +126,11 @@ class qam_nogui(gr.top_block, Qt.QWidget):
         self.connect((self.blocks_throttle_0, 0), (self.channels_channel_model_0, 0))
         self.connect((self.channels_channel_model_0, 0), (self.digital_pfb_clock_sync_xxx_0, 0))
         self.connect((self.digital_cma_equalizer_cc_0, 0), (self.digital_costas_loop_cc_0, 0))
-        self.connect((self.digital_cma_equalizer_cc_0, 0), (self.qtgui_const_sink_x_0, 1))
         self.connect((self.digital_constellation_decoder_cb_0, 0), (self.digital_diff_decoder_bb_0, 0))
         self.connect((self.digital_constellation_modulator_0, 0), (self.blocks_throttle_0, 0))
         self.connect((self.digital_costas_loop_cc_0, 0), (self.digital_constellation_decoder_cb_0, 0))
-        self.connect((self.digital_costas_loop_cc_0, 0), (self.qtgui_const_sink_x_0, 2))
-        self.connect((self.digital_diff_decoder_bb_0, 0), (self.digital_map_bb_0, 0))
-        self.connect((self.digital_map_bb_0, 0), (self.fadingui_xor_frame_sync_0, 0))
+        self.connect((self.digital_diff_decoder_bb_0, 0), (self.fadingui_xor_frame_sync_0, 0))
         self.connect((self.digital_pfb_clock_sync_xxx_0, 0), (self.digital_cma_equalizer_cc_0, 0))
-        self.connect((self.digital_pfb_clock_sync_xxx_0, 0), (self.qtgui_const_sink_x_0, 0))
         self.connect((self.fadingui_datasource_0, 0), (self.digital_constellation_modulator_0, 0))
         self.connect((self.fadingui_deframer_0, 0), (self.blocks_null_sink_0, 0))
         self.connect((self.fadingui_xor_frame_sync_0, 0), (self.fadingui_deframer_0, 0))