Merge branch 'master' of github.com:NaoPross/Fading

13 files changed, 361 insertions, 103 deletions

diff --git a/src/gr-fadingui/grc/CMakeLists.txt b/src/gr-fadingui/grc/CMakeLists.txt
index 1fe2b9c..92b6add 100644
--- a/src/gr-fadingui/grc/CMakeLists.txt
+++ b/src/gr-fadingui/grc/CMakeLists.txt
@@ -20,5 +20,7 @@
 install(FILES
     fadingui_datasource.block.yml
     fadingui_dearpygui_sink.block.yml
-    fadingui_xor_frame_sync.block.yml DESTINATION share/gnuradio/grc/blocks
+    fadingui_xor_frame_sync.block.yml
+    fadingui_deframer.block.yml
+    fadingui_frame_obj.block.yml DESTINATION share/gnuradio/grc/blocks
 )
diff --git a/src/gr-fadingui/grc/fadingui_datasource.block.yml b/src/gr-fadingui/grc/fadingui_datasource.block.yml
index 5f34591..6c31995 100644
--- a/src/gr-fadingui/grc/fadingui_datasource.block.yml
+++ b/src/gr-fadingui/grc/fadingui_datasource.block.yml
@@ -1,10 +1,10 @@
 id: fadingui_datasource
-label: UI Framed Data Source 
+label: Framed Data Source 
 category: '[fadingui]'
 
 templates:
   imports: import fadingui
-  make: fadingui.datasource(vec_len=${vec_len}, header_len=${header_len}, sock_addr=${sock_addr}, file_list=${file_list})
+  make: fadingui.datasource(frame_obj=${frame_obj}, filename=${fname})
 
 #  Make one 'parameters' list entry for every parameter you want settable from the GUI.
 #     Keys include:
@@ -12,25 +12,12 @@ templates:
 #     * label (label shown in the GUI)
 #     * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
 parameters:
-- id: vec_len
-  label: Vector length
-  dtype: int
-  default: 501
-
-- id: header_len
-  label: Header length
-  dtype: int
-  default: 11
-
-- id: sock_addr
-  label: Socket Address
-  dtype: string
-  default: "udp://"
-
-- id: file_list
+- id: frame_obj
+  label: Frame Object
+  type: raw
+- id: fname 
   label: List of files
-  dtype: raw
-  default: "[]"
+  dtype: file_open
 
 #  Make one 'inputs' list entry per input and one 'outputs' list entry per output.
 #  Keys include:
@@ -45,7 +32,6 @@ outputs:
 - label: out
   domain: stream
   dtype: byte
-  vlen: ${ vec_len + header_len }
 
 #  'file_format' specifies the version of the GRC yml format used in the file
 #  and should usually not be changed.
diff --git a/src/gr-fadingui/grc/fadingui_deframer.block.yml b/src/gr-fadingui/grc/fadingui_deframer.block.yml
new file mode 100644
index 0000000..02db246
--- /dev/null
+++ b/src/gr-fadingui/grc/fadingui_deframer.block.yml
@@ -0,0 +1,39 @@
+id: fadingui_deframer
+label: Deframer
+category: '[fadingui]'
+labels: [ python ]
+
+templates:
+  imports: import fadingui
+  make: fadingui.deframer(frame_obj=${frame_obj})
+
+#  Make one 'parameters' list entry for every parameter you want settable from the GUI.
+#     Keys include:
+#     * id (makes the value accessible as \$keyname, e.g. in the make entry)
+#     * label (label shown in the GUI)
+#     * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
+parameters:
+- id: frame_obj
+  label: Frame Object
+  dtype: raw
+
+#  Make one 'inputs' list entry per input and one 'outputs' list entry per output.
+#  Keys include:
+#      * label (an identifier for the GUI)
+#      * domain (optional - stream or message. Default is stream)
+#      * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
+#      * vlen (optional - data stream vector length. Default is 1)
+#      * optional (optional - set to 1 for optional inputs. Default is 0)
+inputs:
+- label: in
+  domain: stream
+  dtype: byte
+
+outputs:
+- label: out
+  domain: stream
+  dtype: byte
+
+#  'file_format' specifies the version of the GRC yml format used in the file
+#  and should usually not be changed.
+file_format: 1
diff --git a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml
new file mode 100644
index 0000000..c655306
--- /dev/null
+++ b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml
@@ -0,0 +1,31 @@
+id: fadingui_frame_obj
+label: Frame Object
+category: '[fadingui]'
+flags: [ show_id, python ]
+
+#  Make one 'parameters' list entry for every parameter you want settable from the GUI.
+#     Keys include:
+#     * id (makes the value accessible as \$keyname, e.g. in the make entry)
+#     * label (label shown in the GUI)
+#     * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
+parameters:
+- id: preamble
+  label: Preamble
+  dtype: int_vector
+  default: '[0xbe, 0xef]'
+- id: payload_len
+  label: Payload length
+  dtype: int
+  default: 4096
+
+value: ${ fadingui.frame_obj }
+
+
+templates:
+  imports: import fadingui
+  var_make: |-
+      self.${id} = ${id} = fadingui.frame_obj(preamble=${preamble}, payload_len=${payload_len})
+
+#  'file_format' specifies the version of the GRC yml format used in the file
+#  and should usually not be changed.
+file_format: 1
diff --git a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml
index 92be2a8..1a8640d 100644
--- a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml
+++ b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml
@@ -1,10 +1,11 @@
 id: fadingui_xor_frame_sync
-label: xor_frame_sync
+label: XOR Correlation Synchronizer
 category: '[fadingui]'
+flags: [ python ]
 
 templates:
   imports: import fadingui
-  make: fadingui.xor_frame_sync()
+  make: fadingui.xor_frame_sync(sync_pattern=${pattern}, buffer_size=${buffer_size})
 
 #  Make one 'parameters' list entry for every parameter you want settable from the GUI.
 #     Keys include:
@@ -12,12 +13,12 @@ templates:
 #     * label (label shown in the GUI)
 #     * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
 parameters:
-- id: ...
-  label: ...
-  dtype: ...
-- id: ...
-  label: ...
-  dtype: ...
+- id: pattern
+  label: Bit pattern
+  dtype: raw
+- id: buffer_size
+  label: Delay buffer size
+  dtype: raw
 
 #  Make one 'inputs' list entry per input and one 'outputs' list entry per output.
 #  Keys include:
@@ -27,18 +28,14 @@ parameters:
 #      * vlen (optional - data stream vector length. Default is 1)
 #      * optional (optional - set to 1 for optional inputs. Default is 0)
 inputs:
-- label: ...
-  domain: ...
-  dtype: ...
-  vlen: ...
-  optional: ...
+- label: in
+  domain: stream
+  dtype: byte
 
 outputs:
-- label: ...
-  domain: ...
-  dtype: ...
-  vlen: ...
-  optional: ...
+- label: out
+  domain: stream
+  dtype: byte
 
 #  'file_format' specifies the version of the GRC yml format used in the file
 #  and should usually not be changed.
diff --git a/src/gr-fadingui/build.sh b/src/gr-fadingui/install.sh
index 29697af..29697af 100755
--- a/src/gr-fadingui/build.sh
+++ b/src/gr-fadingui/install.sh
diff --git a/src/gr-fadingui/python/CMakeLists.txt b/src/gr-fadingui/python/CMakeLists.txt
index 4845bd9..1318857 100644
--- a/src/gr-fadingui/python/CMakeLists.txt
+++ b/src/gr-fadingui/python/CMakeLists.txt
@@ -32,9 +32,12 @@ endif()
 GR_PYTHON_INSTALL(
     FILES
     __init__.py
+    logger.py
     datasource.py
     dearpygui_sink.py
-    xor_frame_sync.py DESTINATION ${GR_PYTHON_DIR}/fadingui
+    xor_frame_sync.py
+    deframer.py
+    frame_obj.py DESTINATION ${GR_PYTHON_DIR}/fadingui
 )
 
 ########################################################################
diff --git a/src/gr-fadingui/python/__init__.py b/src/gr-fadingui/python/__init__.py
index f62e3cf..5fdfea4 100644
--- a/src/gr-fadingui/python/__init__.py
+++ b/src/gr-fadingui/python/__init__.py
@@ -35,5 +35,7 @@ except ImportError:
 from .datasource import datasource
 from .dearpygui_sink import dearpygui_sink
 from .xor_frame_sync import xor_frame_sync
+from .deframer import deframer
+from .frame_obj import frame_obj
 
 #
diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py
index 764b4d5..69a2122 100644
--- a/src/gr-fadingui/python/datasource.py
+++ b/src/gr-fadingui/python/datasource.py
@@ -8,86 +8,75 @@ import io
 import numpy as np
 from gnuradio import gr
 
-class datasource(gr.sync_block):
-    """
-    Loads data from a file choosen in the graphical user interface.
-    """
+from fadingui.logger import get_logger
+log = get_logger("datasource")
 
-    HEADER_LEN = 11;
 
-    def __init__(self, vec_len, header_len, sock_addr, file_list):
-        # FIXME: find a better solution
-        assert(header_len == datasource.HEADER_LEN)
+class datasource(gr.basic_block):
+    """
+    Loads data from a file choosen splits into chunks and pack them into
+    frames.
+    """
 
-        gr.sync_block.__init__(self,
+    def __init__(self, frame_obj, filename):
+        gr.basic_block.__init__(self,
             name="datasource",
             in_sig=None,
-            out_sig=[np.dtype(f'{vec_len + header_len}b')])
+            out_sig=[np.byte])
 
-        # parameters
-        self.vec_len = vec_len
-        self.sock_addr = sock_addr
-        self.file_list = file_list
+        # Frame object
+        self.frame = frame_obj
 
         # file members
-        self.fdata = None
-        self.fsize = None
-        self.fpos = 0
-
-        # cache
-        self.header_cache = None
-
-        # TODO: make it possible to choose from UI
-        self.load_file(file_list[0])
-
-    def load_file(self, fname):
-        self.fdata = np.fromfile(fname, np.byte)
+        self.fname = filename
+        self.fdata = np.fromfile(self.fname, np.byte)
         self.fsize = len(self.fdata)
 
-        # TODO: remove debugging statements or create logger
-        print(f"datasource: loaded file size={self.fsize}, head:")
-        print(self.fdata[:10])
+        # a frame has 5 id bits so, there can only be 2 ** 5 chunks per file
+        # see docstring of frame_obj for more details
+        nblocks = int(self.fsize / self.frame.payload_length)
+        log.debug(f"Loaded {self.fsize} bytes = {nblocks} blocks from name={self.fname}")
+        assert nblocks < 2 ** 5, "Payload size too small or file too big"
 
-    def make_header(self, data_size):
-        # TODO: check that data_size is not too big
-        pilot = 0x1248
+        self.fpos = 0
+        self.blocknr = 0
 
-        # TODO: implement hamming code for header
-        header = f"p{pilot:04x}s{data_size:04x}d".encode("ascii")
+        # would have been nice to have but does not work
+        # self.set_min_noutput_items(frame_obj.length)
 
-        arr = np.frombuffer(header, dtype=np.dtype("byte"))
-        return arr
+        # FIXME: implement buffering
+        # output buffer
+        self.outbuffer = np.array([])
 
-    def work(self, input_items, output_items):
+    def general_work(self, input_items, output_items):
         out = output_items[0]
 
-        if self.fpos + self.vec_len > self.fsize:
-            # FIXME: repair broken code below
-            # TODO: create logger
-            print(f"WARNING: the last {self.fsize - self.fpos} bytes were not sent!")
-            self.fpos = 0
-            return 0;
 
-            rest = self.fsize - self.fpos
+        # FIXME: if there is leftover buffer add that first
+        # if self.outbuffer.size > 0:
+        #     log.debug("Frame did not fit into buffer")
+        #     out[:len(self.outbuffer)] = self.outbuffer
 
-            # cannot use cached header
-            header = self.make_header(rest)
-            data = self.fdata[self.fpos:rest]
+        if self.fpos + self.frame.payload_length > self.fsize:
+            # FIXME: implement edge case
+            log.warning(f"The last {self.fsize - self.fpos} bytes were not sent!")
+            self.fpos = 0
+            self.blocknr = 0
 
-            frame_size = datasource.HEADER_LEN + rest
-            out[:] = np.concatenate([header, data])
+            log.debug("File finished, starting over")
+            return 0;
 
-            self.fpos = 0
-            return rest
+        data = self.fdata[self.fpos:self.fpos + self.frame.payload_length]
+        frame_bytes = self.frame.make(self.blocknr, self.frame.payload_length, data)
 
-        # cache header if not saved
-        if self.header_cache == None:
-            self.header = self.make_header(self.vec_len)
+        out[:] = frame_bytes[:len(out)]
+        self.outbuffer = frame_bytes[len(out):]
 
-        data = self.fdata[self.fpos:self.fpos + self.vec_len]
+        log.debug(f"Sent frame nr={self.blocknr}")
+        log.debug(f"Set bytes {out}")
 
-        out[:] = np.concatenate([self.header, data])
+        self.fpos += self.frame.payload_length
+        self.blocknr += 1
 
-        self.fpos += self.vec_len
-        return len(output_items[0])
+        return self.frame.length
 
diff --git a/src/gr-fadingui/python/deframer.py b/src/gr-fadingui/python/deframer.py
new file mode 100644
index 0000000..2af5ee0
--- /dev/null
+++ b/src/gr-fadingui/python/deframer.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2021 Naoki Pross.
+
+import numpy as np
+from gnuradio import gr
+
+class deframer(gr.sync_block):
+    """
+    Check for integrity and remove frame header from packet.
+    """
+    def __init__(self, frame_obj):
+        gr.sync_block.__init__(self,
+            name="deframer",
+            in_sig=[np.byte],
+            out_sig=[np.byte])
+
+
+    def work(self, input_items, output_items):
+        in0 = input_items[0]
+        out = output_items[0]
+
+        out[:] = in0
+        return len(output_items[0])
+
diff --git a/src/gr-fadingui/python/frame_obj.py b/src/gr-fadingui/python/frame_obj.py
new file mode 100644
index 0000000..136fce0
--- /dev/null
+++ b/src/gr-fadingui/python/frame_obj.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2021 Naoki Pross.
+
+from functools import reduce
+import operator as op
+
+import numpy as np
+from gnuradio import gr
+
+
+class frame_obj:
+    """
+    Frame Object: Contains informations about a data frame.
+
+      -------------------------------------------------------------------------------
+      | Preamble | Padding | ID     | Data Length | Parity | Payload | Padding      |
+      | k bytes  | 1 bit   | 5 bits | 21 bits     | 5 bits |         | if necessary |
+      -------------------------------------------------------------------------------
+                           | (31, 26) Hamming EC code      |
+                           ---------------------------------
+
+    - The preamble is user defined.
+    - The ID (5 bits) + Data length (21 bits) together are a 26 bits, followed
+      by 5 parity bits computed using a (31,26) hamming code.
+
+      This constraints the maximum payload size to 2 MiB and the number IDs to
+      32, i.e. max file size is 64 MiB.
+
+    """
+    def __init__(self, preamble, payload_len):
+        self._preamble = np.array(preamble, dtype=np.uint8)
+
+        self._preamble_len = len(self._preamble)
+        self._payload_len = payload_len
+
+    @property
+    def header_length(self) -> int:
+        """Get header length in bytes"""
+        # 4 is the number of bytes for the hamming part
+        return self._preamble_len + 4
+
+    @property
+    def payload_length(self) -> int:
+        return self._payload_len
+
+    @property
+    def length(self) -> int:
+        """Get frame lenght in bytes"""
+        # 8 is the size of the hamming ECC part + 1 bit
+        return self.header_length + self.payload_length
+
+    @property
+    def preamble(self):
+        """Get preamble"""
+        return self._preamble
+
+    def parity(self, bits):
+        """Compute the 5 parity bits for an unpacked array of 26 bits"""
+        assert(len(bits) == 26)
+        # FIXME: does not work
+        # return np.matmul(bits, gen) % 2
+        return np.array([0, 0, 0, 0, 0])
+
+    def make(self, idv, data_len, data):
+        """Make a frame"""
+        # get lower 4 bits of idv
+        idv_bits = np.unpackbits([np.uint8(idv)])[:5]
+
+        # get lower 22 bits of data_len
+        data_len_bytes = list(map(np.uint8, data_len.to_bytes(4, byteorder='little')))
+        data_len_bits = np.unpackbits(data_len_bytes)[:21]
+
+        # compute 5 parity bits
+        metadata = np.concatenate([idv_bits, data_len_bits])
+        parity_bits = self.parity(metadata)
+
+        # add padding
+        hamming_bits = np.concatenate([[0], metadata, parity_bits])
+        assert(len(hamming_bits) == 32)
+
+        # pack 32 bits into 4 bytes and add the rest
+        hamming = np.packbits(hamming_bits)
+        return np.concatenate([self.preamble, hamming, data])
+
+    def syndrome(self, bits):
+        """Compute the syndrome (check Hamming code) for an unpacked array of 31 bits"""
+        assert(len(bits) == 31)
+        return reduce(op.xor, [i for i, bit in enumerate(bits) if bit])
+
diff --git a/src/gr-fadingui/python/logger.py b/src/gr-fadingui/python/logger.py
new file mode 100644
index 0000000..a189aeb
--- /dev/null
+++ b/src/gr-fadingui/python/logger.py
@@ -0,0 +1,14 @@
+import logging
+import sys
+
+formatter = logging.Formatter("[%(name)s] %(levelname)s: %(message)s")
+stdout_handler = logging.StreamHandler(sys.stdout)
+
+def get_logger(module):
+    log = logging.getLogger(module)
+
+    log.addHandler(stdout_handler)
+    stdout_handler.setFormatter(formatter)
+
+    log.setLevel(logging.DEBUG)
+    return log
diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py
index 9d9064f..51af35a 100644
--- a/src/gr-fadingui/python/xor_frame_sync.py
+++ b/src/gr-fadingui/python/xor_frame_sync.py
@@ -4,24 +4,102 @@
 # Copyright 2021 Naoki Pross.
 
 
-import numpy
+import numpy as np
+from numpy_ringbuffer import RingBuffer
+
 from gnuradio import gr
 
+from fadingui.logger import get_logger
+log = get_logger("xor_frame_sync")
+
+
 class xor_frame_sync(gr.sync_block):
     """
-    docstring for block xor_frame_sync
+    Performs a frame synchronization by XOR matching a preamble bit sequence
     """
-    def __init__(self, sync_pattern):
+    def __init__(self, sync_pattern, buffer_size):
+        # TODO: buffer size should be in packets
         gr.sync_block.__init__(self,
             name="xor_frame_sync",
             in_sig=[np.byte],
             out_sig=[np.byte])
 
+        # binary pattern to match
+        self.pattern = np.unpackbits(np.array(sync_pattern, dtype=np.uint8))[::-1]
+        self.nbytes = len(sync_pattern)
+        self.nbits = len(self.pattern)
+
+        log.debug(f"Loaded pattern {self.pattern} length={self.nbits}")
+        assert(self.nbits % 8 == 0)
+
+        # packed buffer to delay the data
+        self.delaybuf = RingBuffer(buffer_size, dtype=np.uint8)
+        self.delay = 0
+
+        # unpacked buffer to compute correlation values, initially filled with zeros
+        self.corrbuf = RingBuffer(self.nbits)
+        self.corrbuf.extend(np.zeros(self.nbits))
+
+        # buffer to store correlation values
+        self.xcorrs = RingBuffer(buffer_size)
+
+        # synchronization state
+        self.synchronized = False
+
+    def xcorrelation(self, v):
+        """
+        Compute the binary correlations between the stored pattern and
+        correlation buffer, while shifting v into the buffer.
+
+        Binary correlation between two bit vectors is just size of the
+        vector(s) minus the number of bits that differ.
+        """
+        v_arr = np.array(v, dtype=np.uint8)
+        for b in np.unpackbits(v_arr):
+            self.corrbuf.appendleft(b)
+            yield self.nbits - np.sum(np.logical_xor(self.corrbuf, self.pattern))
+
     def work(self, input_items, output_items):
+        """
+        Process the inputs, that means:
+
+            - Check that the buffer is synchronized, i.e. there is the sync
+              pattern appears every k bits, where k is the size of the packet.
+
+            - If the buffer is not synchronized, compute a binary cross
+              correlation to find how much the stream should be delayed.
+
+        Notes:
+
+            - Even though the block input is of type np.byte, inp is an array
+              of 255 bytes, probably for performance reasons.
+              TODO: block processing
+        """
         inp = input_items[0]
         out = output_items[0]
 
-        out[:] = inp
+        if not self.synchronized:
+            for v in inp:
+                # compute the cross correlation
+                xcs = self.xcorrelation(v)
+
+                # add cross correlations to buffer and save value
+                self.xcorrs.extend(list(xcs))
+                self.delaybuf.appendleft(v)
+
+            peak = np.argmax(self.xcorrs)
+            if self.xcorrs[peak] == self.nbits:
+                self.delay = peak
+                self.synchronized = True
+                log.debug(f"Synchronized with delay={peak}")
+
+            else:
+                self.synchronized = False
+                log.warning(f"Did not find a peak (max={self.xcorrs[peak]}, should be {self.nbits})")
+
+
+        # return data with delay
+        out[:] = self.delaybuf[self.delay]
 
         return len(output_items[0])