Implement Frame Object

10 files changed, 140 insertions, 137 deletions

diff --git a/src/gr-fadingui/grc/fadingui_datasource.block.yml b/src/gr-fadingui/grc/fadingui_datasource.block.yml
index 5f34591..6c31995 100644
--- a/src/gr-fadingui/grc/fadingui_datasource.block.yml
+++ b/src/gr-fadingui/grc/fadingui_datasource.block.yml
@@ -1,10 +1,10 @@
 id: fadingui_datasource
-label: UI Framed Data Source 
+label: Framed Data Source 
 category: '[fadingui]'
 
 templates:
   imports: import fadingui
-  make: fadingui.datasource(vec_len=${vec_len}, header_len=${header_len}, sock_addr=${sock_addr}, file_list=${file_list})
+  make: fadingui.datasource(frame_obj=${frame_obj}, filename=${fname})
 
 #  Make one 'parameters' list entry for every parameter you want settable from the GUI.
 #     Keys include:
@@ -12,25 +12,12 @@ templates:
 #     * label (label shown in the GUI)
 #     * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
 parameters:
-- id: vec_len
-  label: Vector length
-  dtype: int
-  default: 501
-
-- id: header_len
-  label: Header length
-  dtype: int
-  default: 11
-
-- id: sock_addr
-  label: Socket Address
-  dtype: string
-  default: "udp://"
-
-- id: file_list
+- id: frame_obj
+  label: Frame Object
+  type: raw
+- id: fname 
   label: List of files
-  dtype: raw
-  default: "[]"
+  dtype: file_open
 
 #  Make one 'inputs' list entry per input and one 'outputs' list entry per output.
 #  Keys include:
@@ -45,7 +32,6 @@ outputs:
 - label: out
   domain: stream
   dtype: byte
-  vlen: ${ vec_len + header_len }
 
 #  'file_format' specifies the version of the GRC yml format used in the file
 #  and should usually not be changed.
diff --git a/src/gr-fadingui/grc/fadingui_deframer.block.yml b/src/gr-fadingui/grc/fadingui_deframer.block.yml
index b5d6cfe..02db246 100644
--- a/src/gr-fadingui/grc/fadingui_deframer.block.yml
+++ b/src/gr-fadingui/grc/fadingui_deframer.block.yml
@@ -1,6 +1,7 @@
 id: fadingui_deframer
 label: Deframer
 category: '[fadingui]'
+labels: [ python ]
 
 templates:
   imports: import fadingui
diff --git a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml
index 8fee3a3..c655306 100644
--- a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml
+++ b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml
@@ -1,10 +1,7 @@
 id: fadingui_frame_obj
 label: Frame Object
 category: '[fadingui]'
-
-templates:
-  imports: import fadingui
-  make: fadingui.frame_obj()
+flags: [ show_id, python ]
 
 #  Make one 'parameters' list entry for every parameter you want settable from the GUI.
 #     Keys include:
@@ -14,22 +11,20 @@ templates:
 parameters:
 - id: preamble
   label: Preamble
-  dtype: raw
-  default: [0xbe, 0xef]
+  dtype: int_vector
+  default: '[0xbe, 0xef]'
 - id: payload_len
   label: Payload length
   dtype: int
   default: 4096
 
-#  Make one 'inputs' list entry per input and one 'outputs' list entry per output.
-#  Keys include:
-#      * label (an identifier for the GUI)
-#      * domain (optional - stream or message. Default is stream)
-#      * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...)
-#      * vlen (optional - data stream vector length. Default is 1)
-#      * optional (optional - set to 1 for optional inputs. Default is 0)
-inputs:
-outputs:
+value: ${ fadingui.frame_obj }
+
+
+templates:
+  imports: import fadingui
+  var_make: |-
+      self.${id} = ${id} = fadingui.frame_obj(preamble=${preamble}, payload_len=${payload_len})
 
 #  'file_format' specifies the version of the GRC yml format used in the file
 #  and should usually not be changed.
diff --git a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml
index 35eb132..1a8640d 100644
--- a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml
+++ b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml
@@ -1,6 +1,7 @@
 id: fadingui_xor_frame_sync
 label: XOR Correlation Synchronizer
 category: '[fadingui]'
+flags: [ python ]
 
 templates:
   imports: import fadingui
@@ -15,14 +16,9 @@ parameters:
 - id: pattern
   label: Bit pattern
   dtype: raw
-
-- id: pattern_len
-  label: Pattern length
-  dtype: raw
-
 - id: buffer_size
   label: Delay buffer size
-  dtype: int
+  dtype: raw
 
 #  Make one 'inputs' list entry per input and one 'outputs' list entry per output.
 #  Keys include:
diff --git a/src/gr-fadingui/python/CMakeLists.txt b/src/gr-fadingui/python/CMakeLists.txt
index c277a73..1318857 100644
--- a/src/gr-fadingui/python/CMakeLists.txt
+++ b/src/gr-fadingui/python/CMakeLists.txt
@@ -32,6 +32,7 @@ endif()
 GR_PYTHON_INSTALL(
     FILES
     __init__.py
+    logger.py
     datasource.py
     dearpygui_sink.py
     xor_frame_sync.py
diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py
index c7d68f1..69a2122 100644
--- a/src/gr-fadingui/python/datasource.py
+++ b/src/gr-fadingui/python/datasource.py
@@ -8,88 +8,75 @@ import io
 import numpy as np
 from gnuradio import gr
 
+from fadingui.logger import get_logger
+log = get_logger("datasource")
 
-class datasource(gr.sync_block):
+
+class datasource(gr.basic_block):
     """
-    Loads data from a file choosen in the graphical user interface, splits into
-    chunks and puts a preamble in front of it(frame).
+    Loads data from a file choosen splits into chunks and pack them into
+    frames.
     """
 
-    HEADER_LEN = 11;
-
-    def __init__(self, vec_len, header_len, sock_addr, file_list):
-        # FIXME: find a better solution
-        assert(header_len == datasource.HEADER_LEN)
-
-        gr.sync_block.__init__(self,
+    def __init__(self, frame_obj, filename):
+        gr.basic_block.__init__(self,
             name="datasource",
             in_sig=None,
-            out_sig=[np.dtype(f'{vec_len + header_len}b')])
+            out_sig=[np.byte])
 
-        # parameters
-        self.vec_len = vec_len
-        self.sock_addr = sock_addr
-        self.file_list = file_list
+        # Frame object
+        self.frame = frame_obj
 
         # file members
-        self.fdata = None
-        self.fsize = None
-        self.fpos = 0
-
-        # cache
-        self.header_cache = None
-
-        # TODO: make it possible to choose from UI
-        self.load_file(file_list[0])
-
-    def load_file(self, fname):
-        self.fdata = np.fromfile(fname, np.byte)
+        self.fname = filename
+        self.fdata = np.fromfile(self.fname, np.byte)
         self.fsize = len(self.fdata)
 
-        # TODO: remove debugging statements or create logger
-        print(f"datasource: loaded file size={self.fsize}, head:")
-        print(self.fdata[:10])
+        # a frame has 5 id bits so, there can only be 2 ** 5 chunks per file
+        # see docstring of frame_obj for more details
+        nblocks = int(self.fsize / self.frame.payload_length)
+        log.debug(f"Loaded {self.fsize} bytes = {nblocks} blocks from name={self.fname}")
+        assert nblocks < 2 ** 5, "Payload size too small or file too big"
 
-    def make_header(self, data_size):
-        # TODO: check that data_size is not too big
-        pilot = 0xbeef
+        self.fpos = 0
+        self.blocknr = 0
 
-        # TODO: implement hamming code for header
-        header = f"p{pilot:04x}s{data_size:04x}d".encode("ascii")
+        # would have been nice to have but does not work
+        # self.set_min_noutput_items(frame_obj.length)
 
-        arr = np.frombuffer(header, dtype=np.dtype("byte"))
-        return arr
+        # FIXME: implement buffering
+        # output buffer
+        self.outbuffer = np.array([])
 
-    def work(self, input_items, output_items):
+    def general_work(self, input_items, output_items):
         out = output_items[0]
 
-        if self.fpos + self.vec_len > self.fsize:
-            # FIXME: repair broken code below
-            # TODO: create logger
-            print(f"WARNING: the last {self.fsize - self.fpos} bytes were not sent!")
-            self.fpos = 0
-            return 0;
 
-            rest = self.fsize - self.fpos
+        # FIXME: if there is leftover buffer add that first
+        # if self.outbuffer.size > 0:
+        #     log.debug("Frame did not fit into buffer")
+        #     out[:len(self.outbuffer)] = self.outbuffer
 
-            # cannot use cached header
-            header = self.make_header(rest)
-            data = self.fdata[self.fpos:rest]
+        if self.fpos + self.frame.payload_length > self.fsize:
+            # FIXME: implement edge case
+            log.warning(f"The last {self.fsize - self.fpos} bytes were not sent!")
+            self.fpos = 0
+            self.blocknr = 0
 
-            frame_size = datasource.HEADER_LEN + rest
-            out[:] = np.concatenate([header, data])
+            log.debug("File finished, starting over")
+            return 0;
 
-            self.fpos = 0
-            return rest
+        data = self.fdata[self.fpos:self.fpos + self.frame.payload_length]
+        frame_bytes = self.frame.make(self.blocknr, self.frame.payload_length, data)
 
-        # cache header if not saved
-        if self.header_cache == None:
-            self.header = self.make_header(self.vec_len)
+        out[:] = frame_bytes[:len(out)]
+        self.outbuffer = frame_bytes[len(out):]
 
-        data = self.fdata[self.fpos:self.fpos + self.vec_len]
+        log.debug(f"Sent frame nr={self.blocknr}")
+        log.debug(f"Set bytes {out}")
 
-        out[:] = np.concatenate([self.header, data])
+        self.fpos += self.frame.payload_length
+        self.blocknr += 1
 
-        self.fpos += self.vec_len
-        return len(output_items[0])
+        return self.frame.length
 
diff --git a/src/gr-fadingui/python/deframer.py b/src/gr-fadingui/python/deframer.py
index b7ee663..2af5ee0 100644
--- a/src/gr-fadingui/python/deframer.py
+++ b/src/gr-fadingui/python/deframer.py
@@ -8,9 +8,9 @@ from gnuradio import gr
 
 class deframer(gr.sync_block):
     """
-    docstring for block deframer
+    Check for integrity and remove frame header from packet.
     """
-    def __init__(self):
+    def __init__(self, frame_obj):
         gr.sync_block.__init__(self,
             name="deframer",
             in_sig=[np.byte],
diff --git a/src/gr-fadingui/python/frame_obj.py b/src/gr-fadingui/python/frame_obj.py
index b9dae70..136fce0 100644
--- a/src/gr-fadingui/python/frame_obj.py
+++ b/src/gr-fadingui/python/frame_obj.py
@@ -10,65 +10,82 @@ import numpy as np
 from gnuradio import gr
 
 
-class frame_obj(gr.basic_block):
+class frame_obj:
     """
     Frame Object: Contains informations about a data frame.
 
       -------------------------------------------------------------------------------
       | Preamble | Padding | ID     | Data Length | Parity | Payload | Padding      |
-      | k bytes  | 1 bit   | 4 bits | 22 bits     | 5 bits |         | if necessary |
+      | k bytes  | 1 bit   | 5 bits | 21 bits     | 5 bits |         | if necessary |
       -------------------------------------------------------------------------------
-                           | (31, 26) Hamming code EC      |
-                           -----------------------------------
+                           | (31, 26) Hamming EC code      |
+                           ---------------------------------
 
     - The preamble is user defined.
-    - The ID (11 bits) + Data length (22 bits) together are a 31 bits, followed
-      by 26 parity bits computed using a (31,26) hamming code.
+    - The ID (5 bits) + Data length (21 bits) together are a 26 bits, followed
+      by 5 parity bits computed using a (31,26) hamming code.
 
-      This constraints the maximum payload size to 64 MB and the number IDs to
-      1024 (i.e. max file size is 1 GB, more than enough for many thing)
+      This constraints the maximum payload size to 2 MiB and the number IDs to
+      32, i.e. max file size is 64 MiB.
 
     """
     def __init__(self, preamble, payload_len):
-        gr.basic_block.__init__(self, name="frame_obj",
-                in_sig=None, out_sig=None)
+        self._preamble = np.array(preamble, dtype=np.uint8)
 
-        self.preamble = np.array(preamble, dtype=np.uint8)
+        self._preamble_len = len(self._preamble)
+        self._payload_len = payload_len
 
-        self.preamble_len = len(self.preamble)
-        self.payload_len = payload_len
+    @property
+    def header_length(self) -> int:
+        """Get header length in bytes"""
+        # 4 is the number of bytes for the hamming part
+        return self._preamble_len + 4
+
+    @property
+    def payload_length(self) -> int:
+        return self._payload_len
+
+    @property
+    def length(self) -> int:
+        """Get frame lenght in bytes"""
+        # 8 is the size of the hamming ECC part + 1 bit
+        return self.header_length + self.payload_length
 
     @property
-    def length(self):
-        """Frame lenght in bytes"""
-        return self.preamble_len + self.payload_len + 8
+    def preamble(self):
+        """Get preamble"""
+        return self._preamble
 
     def parity(self, bits):
         """Compute the 5 parity bits for an unpacked array of 26 bits"""
         assert(len(bits) == 26)
         # FIXME: does not work
-        # gen = np.array(
-        #     [[1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0],
-        #      [0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1],
-        #      [0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0],
-        #      [0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0],
-        #      [0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1]],
-        #     dtype=np.uint8)
         # return np.matmul(bits, gen) % 2
         return np.array([0, 0, 0, 0, 0])
 
     def make(self, idv, data_len, data):
         """Make a frame"""
-        return np.concatenate(self.preamble, np.packbits(hamming), data)
+        # get lower 4 bits of idv
+        idv_bits = np.unpackbits([np.uint8(idv)])[:5]
+
+        # get lower 22 bits of data_len
+        data_len_bytes = list(map(np.uint8, data_len.to_bytes(4, byteorder='little')))
+        data_len_bits = np.unpackbits(data_len_bytes)[:21]
+
+        # compute 5 parity bits
+        metadata = np.concatenate([idv_bits, data_len_bits])
+        parity_bits = self.parity(metadata)
+
+        # add padding
+        hamming_bits = np.concatenate([[0], metadata, parity_bits])
+        assert(len(hamming_bits) == 32)
+
+        # pack 32 bits into 4 bytes and add the rest
+        hamming = np.packbits(hamming_bits)
+        return np.concatenate([self.preamble, hamming, data])
 
     def syndrome(self, bits):
         """Compute the syndrome (check Hamming code) for an unpacked array of 31 bits"""
         assert(len(bits) == 31)
         return reduce(op.xor, [i for i, bit in enumerate(bits) if bit])
 
-    def general_work(self, input_items, output_items):
-        """
-        This block has no inputs or output 
-        """
-        return 0;
-
diff --git a/src/gr-fadingui/python/logger.py b/src/gr-fadingui/python/logger.py
new file mode 100644
index 0000000..a189aeb
--- /dev/null
+++ b/src/gr-fadingui/python/logger.py
@@ -0,0 +1,14 @@
+import logging
+import sys
+
+formatter = logging.Formatter("[%(name)s] %(levelname)s: %(message)s")
+stdout_handler = logging.StreamHandler(sys.stdout)
+
+def get_logger(module):
+    log = logging.getLogger(module)
+
+    log.addHandler(stdout_handler)
+    stdout_handler.setFormatter(formatter)
+
+    log.setLevel(logging.DEBUG)
+    return log
diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py
index e8d202c..51af35a 100644
--- a/src/gr-fadingui/python/xor_frame_sync.py
+++ b/src/gr-fadingui/python/xor_frame_sync.py
@@ -9,6 +9,9 @@ from numpy_ringbuffer import RingBuffer
 
 from gnuradio import gr
 
+from fadingui.logger import get_logger
+log = get_logger("xor_frame_sync")
+
 
 class xor_frame_sync(gr.sync_block):
     """
@@ -26,6 +29,7 @@ class xor_frame_sync(gr.sync_block):
         self.nbytes = len(sync_pattern)
         self.nbits = len(self.pattern)
 
+        log.debug(f"Loaded pattern {self.pattern} length={self.nbits}")
         assert(self.nbits % 8 == 0)
 
         # packed buffer to delay the data
@@ -84,12 +88,14 @@ class xor_frame_sync(gr.sync_block):
                 self.delaybuf.appendleft(v)
 
             peak = np.argmax(self.xcorrs)
-            self.delay = peak
-            self.synchronized = True
+            if self.xcorrs[peak] == self.nbits:
+                self.delay = peak
+                self.synchronized = True
+                log.debug(f"Synchronized with delay={peak}")
 
-            if self.xcorrs[peak] != self.nbits:
+            else:
                 self.synchronized = False
-                print(f"Warning! XOR correlation  did not find a peak (max = {self.xcorrs[peak]} should be {self.nbits})")
+                log.warning(f"Did not find a peak (max={self.xcorrs[peak]}, should be {self.nbits})")
 
 
         # return data with delay