From dd795612f0ca0d84e0a17075f8e56d783937bf10 Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Thu, 18 Nov 2021 16:11:24 +0100 Subject: Begin frame sync block --- .../grc/fadingui_xor_frame_sync.block.yml | 37 +++++++------- src/gr-fadingui/python/datasource.py | 3 +- src/gr-fadingui/python/xor_frame_sync.py | 58 ++++++++++++++++++++-- 3 files changed, 76 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml index 92be2a8..35eb132 100644 --- a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml +++ b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml @@ -1,10 +1,10 @@ id: fadingui_xor_frame_sync -label: xor_frame_sync +label: XOR Correlation Synchronizer category: '[fadingui]' templates: imports: import fadingui - make: fadingui.xor_frame_sync() + make: fadingui.xor_frame_sync(sync_pattern=${pattern}, buffer_size=${buffer_size}) # Make one 'parameters' list entry for every parameter you want settable from the GUI. # Keys include: @@ -12,12 +12,17 @@ templates: # * label (label shown in the GUI) # * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) parameters: -- id: ... - label: ... - dtype: ... -- id: ... - label: ... - dtype: ... +- id: pattern + label: Bit pattern + dtype: raw + +- id: pattern_len + label: Pattern length + dtype: raw + +- id: buffer_size + label: Delay buffer size + dtype: int # Make one 'inputs' list entry per input and one 'outputs' list entry per output. # Keys include: @@ -27,18 +32,14 @@ parameters: # * vlen (optional - data stream vector length. Default is 1) # * optional (optional - set to 1 for optional inputs. Default is 0) inputs: -- label: ... - domain: ... - dtype: ... - vlen: ... - optional: ... +- label: in + domain: stream + dtype: byte outputs: -- label: ... - domain: ... - dtype: ... - vlen: ... - optional: ... +- label: out + domain: stream + dtype: byte # 'file_format' specifies the version of the GRC yml format used in the file # and should usually not be changed. diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py index 764b4d5..ab2f441 100644 --- a/src/gr-fadingui/python/datasource.py +++ b/src/gr-fadingui/python/datasource.py @@ -10,7 +10,8 @@ from gnuradio import gr class datasource(gr.sync_block): """ - Loads data from a file choosen in the graphical user interface. + Loads data from a file choosen in the graphical user interface, splits into + chunks and puts a preamble in front of it(frame). """ HEADER_LEN = 11; diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py index 9d9064f..735a031 100644 --- a/src/gr-fadingui/python/xor_frame_sync.py +++ b/src/gr-fadingui/python/xor_frame_sync.py @@ -4,24 +4,76 @@ # Copyright 2021 Naoki Pross. -import numpy +import numpy as np +from numpy_ringbuffer import RingBuffer + from gnuradio import gr + class xor_frame_sync(gr.sync_block): """ docstring for block xor_frame_sync """ - def __init__(self, sync_pattern): + def __init__(self, sync_pattern, buffer_size): gr.sync_block.__init__(self, name="xor_frame_sync", in_sig=[np.byte], out_sig=[np.byte]) + # binary pattern to match + self.pattern = np.array(sync_pattern, dtype=np.dtype("uint8")) + self.nbits = len(sync_pattern) + + # buffer to delay the data + self.delay_fifo = RingBuffer(buffer_size, dtype=np.byte) + + # buffers to store cross correlation data + self.xcorr = RingBuffer(buffer_size, dtype=np.dtype("uint8")) + + # synchronization state + self.synchronized = False + self.delay = 0 + + def xcorrelation(self): + """ + Compute the binary correlation between the stream and the stored + pattern. Binary correlation between two bit vectors is just size of the + vector(s) minus the number of bits that differ. + """ + unpacked = np.unpackbits(self.delay_fifo[0]) + return self.nbits - sum(np.logical_xor(unpacked, self.pattern)) + def work(self, input_items, output_items): + """ + Process the inputs, that means: + + - Check that the buffer is synchronized, i.e. there is the sync + pattern appears every k bits, where k is the size of the packet. + + - If the buffer is not synchronized, compute a binary cross + correlation to find how by much the stream should be delayed. + """ inp = input_items[0] out = output_items[0] - out[:] = inp + # Add data to delay buffer + self.delay_fifo.appendleft(inp) + + # TODO: check for synchronization, else compute + + # Compute correlation + if not self.synchronized: + self.xcorr.append(self.xcorrelation()) + + peak = np.argmax(self.xcorr) + if self.xcorr[peak] != self.nbits: + print(f"Warning! XOR correlation is not perfect (peak value = {self.xcorr[peak]})") + + self.delay = peak + self.synchronized = True + + # return data with delay + out[:] = self.delay_fifo[self.delay] return len(output_items[0]) -- cgit v1.2.1 From d00bb6d29ceae9d2ee958b57549af335e977edc6 Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Thu, 18 Nov 2021 19:07:03 +0100 Subject: Partially implement xor correlator (untested) --- src/gr-fadingui/build.sh | 8 ---- src/gr-fadingui/install.sh | 8 ++++ src/gr-fadingui/python/datasource.py | 3 +- src/gr-fadingui/python/xor_frame_sync.py | 70 ++++++++++++++++++++------------ 4 files changed, 55 insertions(+), 34 deletions(-) delete mode 100755 src/gr-fadingui/build.sh create mode 100755 src/gr-fadingui/install.sh (limited to 'src') diff --git a/src/gr-fadingui/build.sh b/src/gr-fadingui/build.sh deleted file mode 100755 index 29697af..0000000 --- a/src/gr-fadingui/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -mkdir -p build -cmake -DGR_PYTHON_DIR=/usr/lib/python3/dist-packages/ -GNinja -Bbuild -ninja -C build - -# install to system -sudo ninja -C build install diff --git a/src/gr-fadingui/install.sh b/src/gr-fadingui/install.sh new file mode 100755 index 0000000..29697af --- /dev/null +++ b/src/gr-fadingui/install.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +mkdir -p build +cmake -DGR_PYTHON_DIR=/usr/lib/python3/dist-packages/ -GNinja -Bbuild +ninja -C build + +# install to system +sudo ninja -C build install diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py index ab2f441..c7d68f1 100644 --- a/src/gr-fadingui/python/datasource.py +++ b/src/gr-fadingui/python/datasource.py @@ -8,6 +8,7 @@ import io import numpy as np from gnuradio import gr + class datasource(gr.sync_block): """ Loads data from a file choosen in the graphical user interface, splits into @@ -51,7 +52,7 @@ class datasource(gr.sync_block): def make_header(self, data_size): # TODO: check that data_size is not too big - pilot = 0x1248 + pilot = 0xbeef # TODO: implement hamming code for header header = f"p{pilot:04x}s{data_size:04x}d".encode("ascii") diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py index 735a031..af7aa85 100644 --- a/src/gr-fadingui/python/xor_frame_sync.py +++ b/src/gr-fadingui/python/xor_frame_sync.py @@ -12,36 +12,48 @@ from gnuradio import gr class xor_frame_sync(gr.sync_block): """ - docstring for block xor_frame_sync + Performs a frame synchronization by XOR matching a preamble bit sequence """ def __init__(self, sync_pattern, buffer_size): + # TODO: buffer size should be in packets gr.sync_block.__init__(self, name="xor_frame_sync", in_sig=[np.byte], out_sig=[np.byte]) # binary pattern to match - self.pattern = np.array(sync_pattern, dtype=np.dtype("uint8")) - self.nbits = len(sync_pattern) + self.pattern = np.unpackbits(np.array(sync_pattern, dtype=np.uint8))[::-1] + self.nbytes = len(sync_pattern) + self.nbits = len(self.pattern) - # buffer to delay the data - self.delay_fifo = RingBuffer(buffer_size, dtype=np.byte) + assert(self.nbits % 8 == 0) - # buffers to store cross correlation data - self.xcorr = RingBuffer(buffer_size, dtype=np.dtype("uint8")) + # packed buffer to delay the data + self.delaybuf = RingBuffer(buffer_size, dtype=np.uint8) + self.delay = 0 + + # unpacked buffer to compute correlation values, initially filled with zeros + self.corrbuf = RingBuffer(self.nbits) + self.corrbuf.extend(np.zeros(self.nbits)) + + # buffer to store correlation values + self.xcorrs = RingBuffer(buffer_size) # synchronization state self.synchronized = False - self.delay = 0 - def xcorrelation(self): + def xcorrelation(self, v): """ - Compute the binary correlation between the stream and the stored - pattern. Binary correlation between two bit vectors is just size of the + Compute the binary correlations between the stored pattern and + correlation buffer, while shifting v into the buffer. + + Binary correlation between two bit vectors is just size of the vector(s) minus the number of bits that differ. """ - unpacked = np.unpackbits(self.delay_fifo[0]) - return self.nbits - sum(np.logical_xor(unpacked, self.pattern)) + v_arr = np.array(v, dtype=np.uint8) + for b in np.unpackbits(v_arr): + self.corrbuf.appendleft(b) + yield self.nbits - np.sum(np.logical_xor(self.corrbuf, self.pattern)) def work(self, input_items, output_items): """ @@ -51,29 +63,37 @@ class xor_frame_sync(gr.sync_block): pattern appears every k bits, where k is the size of the packet. - If the buffer is not synchronized, compute a binary cross - correlation to find how by much the stream should be delayed. + correlation to find how much the stream should be delayed. + + Notes: + + - Even though the block input is of type np.byte, inp is an array + of 255 bytes, probably for performance reasons. + TODO: block processing """ inp = input_items[0] out = output_items[0] - # Add data to delay buffer - self.delay_fifo.appendleft(inp) - - # TODO: check for synchronization, else compute - - # Compute correlation if not self.synchronized: - self.xcorr.append(self.xcorrelation()) + for v in inp: + # compute the cross correlation + xcs = self.xcorrelation(v) - peak = np.argmax(self.xcorr) - if self.xcorr[peak] != self.nbits: - print(f"Warning! XOR correlation is not perfect (peak value = {self.xcorr[peak]})") + # add cross correlations to buffer and save value + self.xcorrs.extend(list(xcs)) + self.delaybuf.appendleft(v) + peak = np.argmax(self.xcorrs) self.delay = peak self.synchronized = True + if self.xcorrs[peak] != self.nbits: + self.synchronized = False + print(f"Warning! XOR correlation is not perfect (peak value = {self.xcorrs[peak]})") + + # return data with delay - out[:] = self.delay_fifo[self.delay] + out[:] = self.delaybuf[self.delay] return len(output_items[0]) -- cgit v1.2.1 From ac8ef5f69a69f11fd202470da68709cd3006d547 Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Fri, 19 Nov 2021 21:29:45 +0100 Subject: Frames --- src/gr-fadingui/grc/CMakeLists.txt | 4 +- src/gr-fadingui/grc/fadingui_deframer.block.yml | 38 ++++++++++++ src/gr-fadingui/grc/fadingui_frame_obj.block.yml | 36 ++++++++++++ src/gr-fadingui/python/CMakeLists.txt | 4 +- src/gr-fadingui/python/__init__.py | 2 + src/gr-fadingui/python/deframer.py | 26 +++++++++ src/gr-fadingui/python/frame_obj.py | 74 ++++++++++++++++++++++++ src/gr-fadingui/python/xor_frame_sync.py | 2 +- 8 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 src/gr-fadingui/grc/fadingui_deframer.block.yml create mode 100644 src/gr-fadingui/grc/fadingui_frame_obj.block.yml create mode 100644 src/gr-fadingui/python/deframer.py create mode 100644 src/gr-fadingui/python/frame_obj.py (limited to 'src') diff --git a/src/gr-fadingui/grc/CMakeLists.txt b/src/gr-fadingui/grc/CMakeLists.txt index 1fe2b9c..92b6add 100644 --- a/src/gr-fadingui/grc/CMakeLists.txt +++ b/src/gr-fadingui/grc/CMakeLists.txt @@ -20,5 +20,7 @@ install(FILES fadingui_datasource.block.yml fadingui_dearpygui_sink.block.yml - fadingui_xor_frame_sync.block.yml DESTINATION share/gnuradio/grc/blocks + fadingui_xor_frame_sync.block.yml + fadingui_deframer.block.yml + fadingui_frame_obj.block.yml DESTINATION share/gnuradio/grc/blocks ) diff --git a/src/gr-fadingui/grc/fadingui_deframer.block.yml b/src/gr-fadingui/grc/fadingui_deframer.block.yml new file mode 100644 index 0000000..b5d6cfe --- /dev/null +++ b/src/gr-fadingui/grc/fadingui_deframer.block.yml @@ -0,0 +1,38 @@ +id: fadingui_deframer +label: Deframer +category: '[fadingui]' + +templates: + imports: import fadingui + make: fadingui.deframer(frame_obj=${frame_obj}) + +# Make one 'parameters' list entry for every parameter you want settable from the GUI. +# Keys include: +# * id (makes the value accessible as \$keyname, e.g. in the make entry) +# * label (label shown in the GUI) +# * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) +parameters: +- id: frame_obj + label: Frame Object + dtype: raw + +# Make one 'inputs' list entry per input and one 'outputs' list entry per output. +# Keys include: +# * label (an identifier for the GUI) +# * domain (optional - stream or message. Default is stream) +# * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) +# * vlen (optional - data stream vector length. Default is 1) +# * optional (optional - set to 1 for optional inputs. Default is 0) +inputs: +- label: in + domain: stream + dtype: byte + +outputs: +- label: out + domain: stream + dtype: byte + +# 'file_format' specifies the version of the GRC yml format used in the file +# and should usually not be changed. +file_format: 1 diff --git a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml new file mode 100644 index 0000000..8fee3a3 --- /dev/null +++ b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml @@ -0,0 +1,36 @@ +id: fadingui_frame_obj +label: Frame Object +category: '[fadingui]' + +templates: + imports: import fadingui + make: fadingui.frame_obj() + +# Make one 'parameters' list entry for every parameter you want settable from the GUI. +# Keys include: +# * id (makes the value accessible as \$keyname, e.g. in the make entry) +# * label (label shown in the GUI) +# * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) +parameters: +- id: preamble + label: Preamble + dtype: raw + default: [0xbe, 0xef] +- id: payload_len + label: Payload length + dtype: int + default: 4096 + +# Make one 'inputs' list entry per input and one 'outputs' list entry per output. +# Keys include: +# * label (an identifier for the GUI) +# * domain (optional - stream or message. Default is stream) +# * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) +# * vlen (optional - data stream vector length. Default is 1) +# * optional (optional - set to 1 for optional inputs. Default is 0) +inputs: +outputs: + +# 'file_format' specifies the version of the GRC yml format used in the file +# and should usually not be changed. +file_format: 1 diff --git a/src/gr-fadingui/python/CMakeLists.txt b/src/gr-fadingui/python/CMakeLists.txt index 4845bd9..c277a73 100644 --- a/src/gr-fadingui/python/CMakeLists.txt +++ b/src/gr-fadingui/python/CMakeLists.txt @@ -34,7 +34,9 @@ GR_PYTHON_INSTALL( __init__.py datasource.py dearpygui_sink.py - xor_frame_sync.py DESTINATION ${GR_PYTHON_DIR}/fadingui + xor_frame_sync.py + deframer.py + frame_obj.py DESTINATION ${GR_PYTHON_DIR}/fadingui ) ######################################################################## diff --git a/src/gr-fadingui/python/__init__.py b/src/gr-fadingui/python/__init__.py index f62e3cf..5fdfea4 100644 --- a/src/gr-fadingui/python/__init__.py +++ b/src/gr-fadingui/python/__init__.py @@ -35,5 +35,7 @@ except ImportError: from .datasource import datasource from .dearpygui_sink import dearpygui_sink from .xor_frame_sync import xor_frame_sync +from .deframer import deframer +from .frame_obj import frame_obj # diff --git a/src/gr-fadingui/python/deframer.py b/src/gr-fadingui/python/deframer.py new file mode 100644 index 0000000..b7ee663 --- /dev/null +++ b/src/gr-fadingui/python/deframer.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2021 Naoki Pross. + +import numpy as np +from gnuradio import gr + +class deframer(gr.sync_block): + """ + docstring for block deframer + """ + def __init__(self): + gr.sync_block.__init__(self, + name="deframer", + in_sig=[np.byte], + out_sig=[np.byte]) + + + def work(self, input_items, output_items): + in0 = input_items[0] + out = output_items[0] + + out[:] = in0 + return len(output_items[0]) + diff --git a/src/gr-fadingui/python/frame_obj.py b/src/gr-fadingui/python/frame_obj.py new file mode 100644 index 0000000..b9dae70 --- /dev/null +++ b/src/gr-fadingui/python/frame_obj.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2021 Naoki Pross. + +from functools import reduce +import operator as op + +import numpy as np +from gnuradio import gr + + +class frame_obj(gr.basic_block): + """ + Frame Object: Contains informations about a data frame. + + ------------------------------------------------------------------------------- + | Preamble | Padding | ID | Data Length | Parity | Payload | Padding | + | k bytes | 1 bit | 4 bits | 22 bits | 5 bits | | if necessary | + ------------------------------------------------------------------------------- + | (31, 26) Hamming code EC | + ----------------------------------- + + - The preamble is user defined. + - The ID (11 bits) + Data length (22 bits) together are a 31 bits, followed + by 26 parity bits computed using a (31,26) hamming code. + + This constraints the maximum payload size to 64 MB and the number IDs to + 1024 (i.e. max file size is 1 GB, more than enough for many thing) + + """ + def __init__(self, preamble, payload_len): + gr.basic_block.__init__(self, name="frame_obj", + in_sig=None, out_sig=None) + + self.preamble = np.array(preamble, dtype=np.uint8) + + self.preamble_len = len(self.preamble) + self.payload_len = payload_len + + @property + def length(self): + """Frame lenght in bytes""" + return self.preamble_len + self.payload_len + 8 + + def parity(self, bits): + """Compute the 5 parity bits for an unpacked array of 26 bits""" + assert(len(bits) == 26) + # FIXME: does not work + # gen = np.array( + # [[1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0], + # [0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1], + # [0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0], + # [0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0], + # [0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1]], + # dtype=np.uint8) + # return np.matmul(bits, gen) % 2 + return np.array([0, 0, 0, 0, 0]) + + def make(self, idv, data_len, data): + """Make a frame""" + return np.concatenate(self.preamble, np.packbits(hamming), data) + + def syndrome(self, bits): + """Compute the syndrome (check Hamming code) for an unpacked array of 31 bits""" + assert(len(bits) == 31) + return reduce(op.xor, [i for i, bit in enumerate(bits) if bit]) + + def general_work(self, input_items, output_items): + """ + This block has no inputs or output + """ + return 0; + diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py index af7aa85..e8d202c 100644 --- a/src/gr-fadingui/python/xor_frame_sync.py +++ b/src/gr-fadingui/python/xor_frame_sync.py @@ -89,7 +89,7 @@ class xor_frame_sync(gr.sync_block): if self.xcorrs[peak] != self.nbits: self.synchronized = False - print(f"Warning! XOR correlation is not perfect (peak value = {self.xcorrs[peak]})") + print(f"Warning! XOR correlation did not find a peak (max = {self.xcorrs[peak]} should be {self.nbits})") # return data with delay -- cgit v1.2.1 From 2bb4a947d4d55e605e6912dec3ff95fbe541615a Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Sat, 20 Nov 2021 17:05:04 +0100 Subject: Implement Frame Object --- src/gr-fadingui/grc/fadingui_datasource.block.yml | 28 ++---- src/gr-fadingui/grc/fadingui_deframer.block.yml | 1 + src/gr-fadingui/grc/fadingui_frame_obj.block.yml | 25 ++--- .../grc/fadingui_xor_frame_sync.block.yml | 8 +- src/gr-fadingui/python/CMakeLists.txt | 1 + src/gr-fadingui/python/datasource.py | 105 +++++++++------------ src/gr-fadingui/python/deframer.py | 4 +- src/gr-fadingui/python/frame_obj.py | 77 +++++++++------ src/gr-fadingui/python/logger.py | 14 +++ src/gr-fadingui/python/xor_frame_sync.py | 14 ++- 10 files changed, 140 insertions(+), 137 deletions(-) create mode 100644 src/gr-fadingui/python/logger.py (limited to 'src') diff --git a/src/gr-fadingui/grc/fadingui_datasource.block.yml b/src/gr-fadingui/grc/fadingui_datasource.block.yml index 5f34591..6c31995 100644 --- a/src/gr-fadingui/grc/fadingui_datasource.block.yml +++ b/src/gr-fadingui/grc/fadingui_datasource.block.yml @@ -1,10 +1,10 @@ id: fadingui_datasource -label: UI Framed Data Source +label: Framed Data Source category: '[fadingui]' templates: imports: import fadingui - make: fadingui.datasource(vec_len=${vec_len}, header_len=${header_len}, sock_addr=${sock_addr}, file_list=${file_list}) + make: fadingui.datasource(frame_obj=${frame_obj}, filename=${fname}) # Make one 'parameters' list entry for every parameter you want settable from the GUI. # Keys include: @@ -12,25 +12,12 @@ templates: # * label (label shown in the GUI) # * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) parameters: -- id: vec_len - label: Vector length - dtype: int - default: 501 - -- id: header_len - label: Header length - dtype: int - default: 11 - -- id: sock_addr - label: Socket Address - dtype: string - default: "udp://" - -- id: file_list +- id: frame_obj + label: Frame Object + type: raw +- id: fname label: List of files - dtype: raw - default: "[]" + dtype: file_open # Make one 'inputs' list entry per input and one 'outputs' list entry per output. # Keys include: @@ -45,7 +32,6 @@ outputs: - label: out domain: stream dtype: byte - vlen: ${ vec_len + header_len } # 'file_format' specifies the version of the GRC yml format used in the file # and should usually not be changed. diff --git a/src/gr-fadingui/grc/fadingui_deframer.block.yml b/src/gr-fadingui/grc/fadingui_deframer.block.yml index b5d6cfe..02db246 100644 --- a/src/gr-fadingui/grc/fadingui_deframer.block.yml +++ b/src/gr-fadingui/grc/fadingui_deframer.block.yml @@ -1,6 +1,7 @@ id: fadingui_deframer label: Deframer category: '[fadingui]' +labels: [ python ] templates: imports: import fadingui diff --git a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml index 8fee3a3..c655306 100644 --- a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml +++ b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml @@ -1,10 +1,7 @@ id: fadingui_frame_obj label: Frame Object category: '[fadingui]' - -templates: - imports: import fadingui - make: fadingui.frame_obj() +flags: [ show_id, python ] # Make one 'parameters' list entry for every parameter you want settable from the GUI. # Keys include: @@ -14,22 +11,20 @@ templates: parameters: - id: preamble label: Preamble - dtype: raw - default: [0xbe, 0xef] + dtype: int_vector + default: '[0xbe, 0xef]' - id: payload_len label: Payload length dtype: int default: 4096 -# Make one 'inputs' list entry per input and one 'outputs' list entry per output. -# Keys include: -# * label (an identifier for the GUI) -# * domain (optional - stream or message. Default is stream) -# * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) -# * vlen (optional - data stream vector length. Default is 1) -# * optional (optional - set to 1 for optional inputs. Default is 0) -inputs: -outputs: +value: ${ fadingui.frame_obj } + + +templates: + imports: import fadingui + var_make: |- + self.${id} = ${id} = fadingui.frame_obj(preamble=${preamble}, payload_len=${payload_len}) # 'file_format' specifies the version of the GRC yml format used in the file # and should usually not be changed. diff --git a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml index 35eb132..1a8640d 100644 --- a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml +++ b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml @@ -1,6 +1,7 @@ id: fadingui_xor_frame_sync label: XOR Correlation Synchronizer category: '[fadingui]' +flags: [ python ] templates: imports: import fadingui @@ -15,14 +16,9 @@ parameters: - id: pattern label: Bit pattern dtype: raw - -- id: pattern_len - label: Pattern length - dtype: raw - - id: buffer_size label: Delay buffer size - dtype: int + dtype: raw # Make one 'inputs' list entry per input and one 'outputs' list entry per output. # Keys include: diff --git a/src/gr-fadingui/python/CMakeLists.txt b/src/gr-fadingui/python/CMakeLists.txt index c277a73..1318857 100644 --- a/src/gr-fadingui/python/CMakeLists.txt +++ b/src/gr-fadingui/python/CMakeLists.txt @@ -32,6 +32,7 @@ endif() GR_PYTHON_INSTALL( FILES __init__.py + logger.py datasource.py dearpygui_sink.py xor_frame_sync.py diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py index c7d68f1..69a2122 100644 --- a/src/gr-fadingui/python/datasource.py +++ b/src/gr-fadingui/python/datasource.py @@ -8,88 +8,75 @@ import io import numpy as np from gnuradio import gr +from fadingui.logger import get_logger +log = get_logger("datasource") -class datasource(gr.sync_block): + +class datasource(gr.basic_block): """ - Loads data from a file choosen in the graphical user interface, splits into - chunks and puts a preamble in front of it(frame). + Loads data from a file choosen splits into chunks and pack them into + frames. """ - HEADER_LEN = 11; - - def __init__(self, vec_len, header_len, sock_addr, file_list): - # FIXME: find a better solution - assert(header_len == datasource.HEADER_LEN) - - gr.sync_block.__init__(self, + def __init__(self, frame_obj, filename): + gr.basic_block.__init__(self, name="datasource", in_sig=None, - out_sig=[np.dtype(f'{vec_len + header_len}b')]) + out_sig=[np.byte]) - # parameters - self.vec_len = vec_len - self.sock_addr = sock_addr - self.file_list = file_list + # Frame object + self.frame = frame_obj # file members - self.fdata = None - self.fsize = None - self.fpos = 0 - - # cache - self.header_cache = None - - # TODO: make it possible to choose from UI - self.load_file(file_list[0]) - - def load_file(self, fname): - self.fdata = np.fromfile(fname, np.byte) + self.fname = filename + self.fdata = np.fromfile(self.fname, np.byte) self.fsize = len(self.fdata) - # TODO: remove debugging statements or create logger - print(f"datasource: loaded file size={self.fsize}, head:") - print(self.fdata[:10]) + # a frame has 5 id bits so, there can only be 2 ** 5 chunks per file + # see docstring of frame_obj for more details + nblocks = int(self.fsize / self.frame.payload_length) + log.debug(f"Loaded {self.fsize} bytes = {nblocks} blocks from name={self.fname}") + assert nblocks < 2 ** 5, "Payload size too small or file too big" - def make_header(self, data_size): - # TODO: check that data_size is not too big - pilot = 0xbeef + self.fpos = 0 + self.blocknr = 0 - # TODO: implement hamming code for header - header = f"p{pilot:04x}s{data_size:04x}d".encode("ascii") + # would have been nice to have but does not work + # self.set_min_noutput_items(frame_obj.length) - arr = np.frombuffer(header, dtype=np.dtype("byte")) - return arr + # FIXME: implement buffering + # output buffer + self.outbuffer = np.array([]) - def work(self, input_items, output_items): + def general_work(self, input_items, output_items): out = output_items[0] - if self.fpos + self.vec_len > self.fsize: - # FIXME: repair broken code below - # TODO: create logger - print(f"WARNING: the last {self.fsize - self.fpos} bytes were not sent!") - self.fpos = 0 - return 0; - rest = self.fsize - self.fpos + # FIXME: if there is leftover buffer add that first + # if self.outbuffer.size > 0: + # log.debug("Frame did not fit into buffer") + # out[:len(self.outbuffer)] = self.outbuffer - # cannot use cached header - header = self.make_header(rest) - data = self.fdata[self.fpos:rest] + if self.fpos + self.frame.payload_length > self.fsize: + # FIXME: implement edge case + log.warning(f"The last {self.fsize - self.fpos} bytes were not sent!") + self.fpos = 0 + self.blocknr = 0 - frame_size = datasource.HEADER_LEN + rest - out[:] = np.concatenate([header, data]) + log.debug("File finished, starting over") + return 0; - self.fpos = 0 - return rest + data = self.fdata[self.fpos:self.fpos + self.frame.payload_length] + frame_bytes = self.frame.make(self.blocknr, self.frame.payload_length, data) - # cache header if not saved - if self.header_cache == None: - self.header = self.make_header(self.vec_len) + out[:] = frame_bytes[:len(out)] + self.outbuffer = frame_bytes[len(out):] - data = self.fdata[self.fpos:self.fpos + self.vec_len] + log.debug(f"Sent frame nr={self.blocknr}") + log.debug(f"Set bytes {out}") - out[:] = np.concatenate([self.header, data]) + self.fpos += self.frame.payload_length + self.blocknr += 1 - self.fpos += self.vec_len - return len(output_items[0]) + return self.frame.length diff --git a/src/gr-fadingui/python/deframer.py b/src/gr-fadingui/python/deframer.py index b7ee663..2af5ee0 100644 --- a/src/gr-fadingui/python/deframer.py +++ b/src/gr-fadingui/python/deframer.py @@ -8,9 +8,9 @@ from gnuradio import gr class deframer(gr.sync_block): """ - docstring for block deframer + Check for integrity and remove frame header from packet. """ - def __init__(self): + def __init__(self, frame_obj): gr.sync_block.__init__(self, name="deframer", in_sig=[np.byte], diff --git a/src/gr-fadingui/python/frame_obj.py b/src/gr-fadingui/python/frame_obj.py index b9dae70..136fce0 100644 --- a/src/gr-fadingui/python/frame_obj.py +++ b/src/gr-fadingui/python/frame_obj.py @@ -10,65 +10,82 @@ import numpy as np from gnuradio import gr -class frame_obj(gr.basic_block): +class frame_obj: """ Frame Object: Contains informations about a data frame. ------------------------------------------------------------------------------- | Preamble | Padding | ID | Data Length | Parity | Payload | Padding | - | k bytes | 1 bit | 4 bits | 22 bits | 5 bits | | if necessary | + | k bytes | 1 bit | 5 bits | 21 bits | 5 bits | | if necessary | ------------------------------------------------------------------------------- - | (31, 26) Hamming code EC | - ----------------------------------- + | (31, 26) Hamming EC code | + --------------------------------- - The preamble is user defined. - - The ID (11 bits) + Data length (22 bits) together are a 31 bits, followed - by 26 parity bits computed using a (31,26) hamming code. + - The ID (5 bits) + Data length (21 bits) together are a 26 bits, followed + by 5 parity bits computed using a (31,26) hamming code. - This constraints the maximum payload size to 64 MB and the number IDs to - 1024 (i.e. max file size is 1 GB, more than enough for many thing) + This constraints the maximum payload size to 2 MiB and the number IDs to + 32, i.e. max file size is 64 MiB. """ def __init__(self, preamble, payload_len): - gr.basic_block.__init__(self, name="frame_obj", - in_sig=None, out_sig=None) + self._preamble = np.array(preamble, dtype=np.uint8) - self.preamble = np.array(preamble, dtype=np.uint8) + self._preamble_len = len(self._preamble) + self._payload_len = payload_len - self.preamble_len = len(self.preamble) - self.payload_len = payload_len + @property + def header_length(self) -> int: + """Get header length in bytes""" + # 4 is the number of bytes for the hamming part + return self._preamble_len + 4 + + @property + def payload_length(self) -> int: + return self._payload_len + + @property + def length(self) -> int: + """Get frame lenght in bytes""" + # 8 is the size of the hamming ECC part + 1 bit + return self.header_length + self.payload_length @property - def length(self): - """Frame lenght in bytes""" - return self.preamble_len + self.payload_len + 8 + def preamble(self): + """Get preamble""" + return self._preamble def parity(self, bits): """Compute the 5 parity bits for an unpacked array of 26 bits""" assert(len(bits) == 26) # FIXME: does not work - # gen = np.array( - # [[1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0], - # [0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1], - # [0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0], - # [0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0], - # [0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1]], - # dtype=np.uint8) # return np.matmul(bits, gen) % 2 return np.array([0, 0, 0, 0, 0]) def make(self, idv, data_len, data): """Make a frame""" - return np.concatenate(self.preamble, np.packbits(hamming), data) + # get lower 4 bits of idv + idv_bits = np.unpackbits([np.uint8(idv)])[:5] + + # get lower 22 bits of data_len + data_len_bytes = list(map(np.uint8, data_len.to_bytes(4, byteorder='little'))) + data_len_bits = np.unpackbits(data_len_bytes)[:21] + + # compute 5 parity bits + metadata = np.concatenate([idv_bits, data_len_bits]) + parity_bits = self.parity(metadata) + + # add padding + hamming_bits = np.concatenate([[0], metadata, parity_bits]) + assert(len(hamming_bits) == 32) + + # pack 32 bits into 4 bytes and add the rest + hamming = np.packbits(hamming_bits) + return np.concatenate([self.preamble, hamming, data]) def syndrome(self, bits): """Compute the syndrome (check Hamming code) for an unpacked array of 31 bits""" assert(len(bits) == 31) return reduce(op.xor, [i for i, bit in enumerate(bits) if bit]) - def general_work(self, input_items, output_items): - """ - This block has no inputs or output - """ - return 0; - diff --git a/src/gr-fadingui/python/logger.py b/src/gr-fadingui/python/logger.py new file mode 100644 index 0000000..a189aeb --- /dev/null +++ b/src/gr-fadingui/python/logger.py @@ -0,0 +1,14 @@ +import logging +import sys + +formatter = logging.Formatter("[%(name)s] %(levelname)s: %(message)s") +stdout_handler = logging.StreamHandler(sys.stdout) + +def get_logger(module): + log = logging.getLogger(module) + + log.addHandler(stdout_handler) + stdout_handler.setFormatter(formatter) + + log.setLevel(logging.DEBUG) + return log diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py index e8d202c..51af35a 100644 --- a/src/gr-fadingui/python/xor_frame_sync.py +++ b/src/gr-fadingui/python/xor_frame_sync.py @@ -9,6 +9,9 @@ from numpy_ringbuffer import RingBuffer from gnuradio import gr +from fadingui.logger import get_logger +log = get_logger("xor_frame_sync") + class xor_frame_sync(gr.sync_block): """ @@ -26,6 +29,7 @@ class xor_frame_sync(gr.sync_block): self.nbytes = len(sync_pattern) self.nbits = len(self.pattern) + log.debug(f"Loaded pattern {self.pattern} length={self.nbits}") assert(self.nbits % 8 == 0) # packed buffer to delay the data @@ -84,12 +88,14 @@ class xor_frame_sync(gr.sync_block): self.delaybuf.appendleft(v) peak = np.argmax(self.xcorrs) - self.delay = peak - self.synchronized = True + if self.xcorrs[peak] == self.nbits: + self.delay = peak + self.synchronized = True + log.debug(f"Synchronized with delay={peak}") - if self.xcorrs[peak] != self.nbits: + else: self.synchronized = False - print(f"Warning! XOR correlation did not find a peak (max = {self.xcorrs[peak]} should be {self.nbits})") + log.warning(f"Did not find a peak (max={self.xcorrs[peak]}, should be {self.nbits})") # return data with delay -- cgit v1.2.1