From 2bb4a947d4d55e605e6912dec3ff95fbe541615a Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Sat, 20 Nov 2021 17:05:04 +0100 Subject: Implement Frame Object --- src/gr-fadingui/grc/fadingui_datasource.block.yml | 28 ++---- src/gr-fadingui/grc/fadingui_deframer.block.yml | 1 + src/gr-fadingui/grc/fadingui_frame_obj.block.yml | 25 ++--- .../grc/fadingui_xor_frame_sync.block.yml | 8 +- src/gr-fadingui/python/CMakeLists.txt | 1 + src/gr-fadingui/python/datasource.py | 105 +++++++++------------ src/gr-fadingui/python/deframer.py | 4 +- src/gr-fadingui/python/frame_obj.py | 77 +++++++++------ src/gr-fadingui/python/logger.py | 14 +++ src/gr-fadingui/python/xor_frame_sync.py | 14 ++- 10 files changed, 140 insertions(+), 137 deletions(-) create mode 100644 src/gr-fadingui/python/logger.py (limited to 'src') diff --git a/src/gr-fadingui/grc/fadingui_datasource.block.yml b/src/gr-fadingui/grc/fadingui_datasource.block.yml index 5f34591..6c31995 100644 --- a/src/gr-fadingui/grc/fadingui_datasource.block.yml +++ b/src/gr-fadingui/grc/fadingui_datasource.block.yml @@ -1,10 +1,10 @@ id: fadingui_datasource -label: UI Framed Data Source +label: Framed Data Source category: '[fadingui]' templates: imports: import fadingui - make: fadingui.datasource(vec_len=${vec_len}, header_len=${header_len}, sock_addr=${sock_addr}, file_list=${file_list}) + make: fadingui.datasource(frame_obj=${frame_obj}, filename=${fname}) # Make one 'parameters' list entry for every parameter you want settable from the GUI. # Keys include: @@ -12,25 +12,12 @@ templates: # * label (label shown in the GUI) # * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) parameters: -- id: vec_len - label: Vector length - dtype: int - default: 501 - -- id: header_len - label: Header length - dtype: int - default: 11 - -- id: sock_addr - label: Socket Address - dtype: string - default: "udp://" - -- id: file_list +- id: frame_obj + label: Frame Object + type: raw +- id: fname label: List of files - dtype: raw - default: "[]" + dtype: file_open # Make one 'inputs' list entry per input and one 'outputs' list entry per output. # Keys include: @@ -45,7 +32,6 @@ outputs: - label: out domain: stream dtype: byte - vlen: ${ vec_len + header_len } # 'file_format' specifies the version of the GRC yml format used in the file # and should usually not be changed. diff --git a/src/gr-fadingui/grc/fadingui_deframer.block.yml b/src/gr-fadingui/grc/fadingui_deframer.block.yml index b5d6cfe..02db246 100644 --- a/src/gr-fadingui/grc/fadingui_deframer.block.yml +++ b/src/gr-fadingui/grc/fadingui_deframer.block.yml @@ -1,6 +1,7 @@ id: fadingui_deframer label: Deframer category: '[fadingui]' +labels: [ python ] templates: imports: import fadingui diff --git a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml index 8fee3a3..c655306 100644 --- a/src/gr-fadingui/grc/fadingui_frame_obj.block.yml +++ b/src/gr-fadingui/grc/fadingui_frame_obj.block.yml @@ -1,10 +1,7 @@ id: fadingui_frame_obj label: Frame Object category: '[fadingui]' - -templates: - imports: import fadingui - make: fadingui.frame_obj() +flags: [ show_id, python ] # Make one 'parameters' list entry for every parameter you want settable from the GUI. # Keys include: @@ -14,22 +11,20 @@ templates: parameters: - id: preamble label: Preamble - dtype: raw - default: [0xbe, 0xef] + dtype: int_vector + default: '[0xbe, 0xef]' - id: payload_len label: Payload length dtype: int default: 4096 -# Make one 'inputs' list entry per input and one 'outputs' list entry per output. -# Keys include: -# * label (an identifier for the GUI) -# * domain (optional - stream or message. Default is stream) -# * dtype (e.g. int, float, complex, byte, short, xxx_vector, ...) -# * vlen (optional - data stream vector length. Default is 1) -# * optional (optional - set to 1 for optional inputs. Default is 0) -inputs: -outputs: +value: ${ fadingui.frame_obj } + + +templates: + imports: import fadingui + var_make: |- + self.${id} = ${id} = fadingui.frame_obj(preamble=${preamble}, payload_len=${payload_len}) # 'file_format' specifies the version of the GRC yml format used in the file # and should usually not be changed. diff --git a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml index 35eb132..1a8640d 100644 --- a/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml +++ b/src/gr-fadingui/grc/fadingui_xor_frame_sync.block.yml @@ -1,6 +1,7 @@ id: fadingui_xor_frame_sync label: XOR Correlation Synchronizer category: '[fadingui]' +flags: [ python ] templates: imports: import fadingui @@ -15,14 +16,9 @@ parameters: - id: pattern label: Bit pattern dtype: raw - -- id: pattern_len - label: Pattern length - dtype: raw - - id: buffer_size label: Delay buffer size - dtype: int + dtype: raw # Make one 'inputs' list entry per input and one 'outputs' list entry per output. # Keys include: diff --git a/src/gr-fadingui/python/CMakeLists.txt b/src/gr-fadingui/python/CMakeLists.txt index c277a73..1318857 100644 --- a/src/gr-fadingui/python/CMakeLists.txt +++ b/src/gr-fadingui/python/CMakeLists.txt @@ -32,6 +32,7 @@ endif() GR_PYTHON_INSTALL( FILES __init__.py + logger.py datasource.py dearpygui_sink.py xor_frame_sync.py diff --git a/src/gr-fadingui/python/datasource.py b/src/gr-fadingui/python/datasource.py index c7d68f1..69a2122 100644 --- a/src/gr-fadingui/python/datasource.py +++ b/src/gr-fadingui/python/datasource.py @@ -8,88 +8,75 @@ import io import numpy as np from gnuradio import gr +from fadingui.logger import get_logger +log = get_logger("datasource") -class datasource(gr.sync_block): + +class datasource(gr.basic_block): """ - Loads data from a file choosen in the graphical user interface, splits into - chunks and puts a preamble in front of it(frame). + Loads data from a file choosen splits into chunks and pack them into + frames. """ - HEADER_LEN = 11; - - def __init__(self, vec_len, header_len, sock_addr, file_list): - # FIXME: find a better solution - assert(header_len == datasource.HEADER_LEN) - - gr.sync_block.__init__(self, + def __init__(self, frame_obj, filename): + gr.basic_block.__init__(self, name="datasource", in_sig=None, - out_sig=[np.dtype(f'{vec_len + header_len}b')]) + out_sig=[np.byte]) - # parameters - self.vec_len = vec_len - self.sock_addr = sock_addr - self.file_list = file_list + # Frame object + self.frame = frame_obj # file members - self.fdata = None - self.fsize = None - self.fpos = 0 - - # cache - self.header_cache = None - - # TODO: make it possible to choose from UI - self.load_file(file_list[0]) - - def load_file(self, fname): - self.fdata = np.fromfile(fname, np.byte) + self.fname = filename + self.fdata = np.fromfile(self.fname, np.byte) self.fsize = len(self.fdata) - # TODO: remove debugging statements or create logger - print(f"datasource: loaded file size={self.fsize}, head:") - print(self.fdata[:10]) + # a frame has 5 id bits so, there can only be 2 ** 5 chunks per file + # see docstring of frame_obj for more details + nblocks = int(self.fsize / self.frame.payload_length) + log.debug(f"Loaded {self.fsize} bytes = {nblocks} blocks from name={self.fname}") + assert nblocks < 2 ** 5, "Payload size too small or file too big" - def make_header(self, data_size): - # TODO: check that data_size is not too big - pilot = 0xbeef + self.fpos = 0 + self.blocknr = 0 - # TODO: implement hamming code for header - header = f"p{pilot:04x}s{data_size:04x}d".encode("ascii") + # would have been nice to have but does not work + # self.set_min_noutput_items(frame_obj.length) - arr = np.frombuffer(header, dtype=np.dtype("byte")) - return arr + # FIXME: implement buffering + # output buffer + self.outbuffer = np.array([]) - def work(self, input_items, output_items): + def general_work(self, input_items, output_items): out = output_items[0] - if self.fpos + self.vec_len > self.fsize: - # FIXME: repair broken code below - # TODO: create logger - print(f"WARNING: the last {self.fsize - self.fpos} bytes were not sent!") - self.fpos = 0 - return 0; - rest = self.fsize - self.fpos + # FIXME: if there is leftover buffer add that first + # if self.outbuffer.size > 0: + # log.debug("Frame did not fit into buffer") + # out[:len(self.outbuffer)] = self.outbuffer - # cannot use cached header - header = self.make_header(rest) - data = self.fdata[self.fpos:rest] + if self.fpos + self.frame.payload_length > self.fsize: + # FIXME: implement edge case + log.warning(f"The last {self.fsize - self.fpos} bytes were not sent!") + self.fpos = 0 + self.blocknr = 0 - frame_size = datasource.HEADER_LEN + rest - out[:] = np.concatenate([header, data]) + log.debug("File finished, starting over") + return 0; - self.fpos = 0 - return rest + data = self.fdata[self.fpos:self.fpos + self.frame.payload_length] + frame_bytes = self.frame.make(self.blocknr, self.frame.payload_length, data) - # cache header if not saved - if self.header_cache == None: - self.header = self.make_header(self.vec_len) + out[:] = frame_bytes[:len(out)] + self.outbuffer = frame_bytes[len(out):] - data = self.fdata[self.fpos:self.fpos + self.vec_len] + log.debug(f"Sent frame nr={self.blocknr}") + log.debug(f"Set bytes {out}") - out[:] = np.concatenate([self.header, data]) + self.fpos += self.frame.payload_length + self.blocknr += 1 - self.fpos += self.vec_len - return len(output_items[0]) + return self.frame.length diff --git a/src/gr-fadingui/python/deframer.py b/src/gr-fadingui/python/deframer.py index b7ee663..2af5ee0 100644 --- a/src/gr-fadingui/python/deframer.py +++ b/src/gr-fadingui/python/deframer.py @@ -8,9 +8,9 @@ from gnuradio import gr class deframer(gr.sync_block): """ - docstring for block deframer + Check for integrity and remove frame header from packet. """ - def __init__(self): + def __init__(self, frame_obj): gr.sync_block.__init__(self, name="deframer", in_sig=[np.byte], diff --git a/src/gr-fadingui/python/frame_obj.py b/src/gr-fadingui/python/frame_obj.py index b9dae70..136fce0 100644 --- a/src/gr-fadingui/python/frame_obj.py +++ b/src/gr-fadingui/python/frame_obj.py @@ -10,65 +10,82 @@ import numpy as np from gnuradio import gr -class frame_obj(gr.basic_block): +class frame_obj: """ Frame Object: Contains informations about a data frame. ------------------------------------------------------------------------------- | Preamble | Padding | ID | Data Length | Parity | Payload | Padding | - | k bytes | 1 bit | 4 bits | 22 bits | 5 bits | | if necessary | + | k bytes | 1 bit | 5 bits | 21 bits | 5 bits | | if necessary | ------------------------------------------------------------------------------- - | (31, 26) Hamming code EC | - ----------------------------------- + | (31, 26) Hamming EC code | + --------------------------------- - The preamble is user defined. - - The ID (11 bits) + Data length (22 bits) together are a 31 bits, followed - by 26 parity bits computed using a (31,26) hamming code. + - The ID (5 bits) + Data length (21 bits) together are a 26 bits, followed + by 5 parity bits computed using a (31,26) hamming code. - This constraints the maximum payload size to 64 MB and the number IDs to - 1024 (i.e. max file size is 1 GB, more than enough for many thing) + This constraints the maximum payload size to 2 MiB and the number IDs to + 32, i.e. max file size is 64 MiB. """ def __init__(self, preamble, payload_len): - gr.basic_block.__init__(self, name="frame_obj", - in_sig=None, out_sig=None) + self._preamble = np.array(preamble, dtype=np.uint8) - self.preamble = np.array(preamble, dtype=np.uint8) + self._preamble_len = len(self._preamble) + self._payload_len = payload_len - self.preamble_len = len(self.preamble) - self.payload_len = payload_len + @property + def header_length(self) -> int: + """Get header length in bytes""" + # 4 is the number of bytes for the hamming part + return self._preamble_len + 4 + + @property + def payload_length(self) -> int: + return self._payload_len + + @property + def length(self) -> int: + """Get frame lenght in bytes""" + # 8 is the size of the hamming ECC part + 1 bit + return self.header_length + self.payload_length @property - def length(self): - """Frame lenght in bytes""" - return self.preamble_len + self.payload_len + 8 + def preamble(self): + """Get preamble""" + return self._preamble def parity(self, bits): """Compute the 5 parity bits for an unpacked array of 26 bits""" assert(len(bits) == 26) # FIXME: does not work - # gen = np.array( - # [[1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0], - # [0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1], - # [0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0], - # [0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0], - # [0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1]], - # dtype=np.uint8) # return np.matmul(bits, gen) % 2 return np.array([0, 0, 0, 0, 0]) def make(self, idv, data_len, data): """Make a frame""" - return np.concatenate(self.preamble, np.packbits(hamming), data) + # get lower 4 bits of idv + idv_bits = np.unpackbits([np.uint8(idv)])[:5] + + # get lower 22 bits of data_len + data_len_bytes = list(map(np.uint8, data_len.to_bytes(4, byteorder='little'))) + data_len_bits = np.unpackbits(data_len_bytes)[:21] + + # compute 5 parity bits + metadata = np.concatenate([idv_bits, data_len_bits]) + parity_bits = self.parity(metadata) + + # add padding + hamming_bits = np.concatenate([[0], metadata, parity_bits]) + assert(len(hamming_bits) == 32) + + # pack 32 bits into 4 bytes and add the rest + hamming = np.packbits(hamming_bits) + return np.concatenate([self.preamble, hamming, data]) def syndrome(self, bits): """Compute the syndrome (check Hamming code) for an unpacked array of 31 bits""" assert(len(bits) == 31) return reduce(op.xor, [i for i, bit in enumerate(bits) if bit]) - def general_work(self, input_items, output_items): - """ - This block has no inputs or output - """ - return 0; - diff --git a/src/gr-fadingui/python/logger.py b/src/gr-fadingui/python/logger.py new file mode 100644 index 0000000..a189aeb --- /dev/null +++ b/src/gr-fadingui/python/logger.py @@ -0,0 +1,14 @@ +import logging +import sys + +formatter = logging.Formatter("[%(name)s] %(levelname)s: %(message)s") +stdout_handler = logging.StreamHandler(sys.stdout) + +def get_logger(module): + log = logging.getLogger(module) + + log.addHandler(stdout_handler) + stdout_handler.setFormatter(formatter) + + log.setLevel(logging.DEBUG) + return log diff --git a/src/gr-fadingui/python/xor_frame_sync.py b/src/gr-fadingui/python/xor_frame_sync.py index e8d202c..51af35a 100644 --- a/src/gr-fadingui/python/xor_frame_sync.py +++ b/src/gr-fadingui/python/xor_frame_sync.py @@ -9,6 +9,9 @@ from numpy_ringbuffer import RingBuffer from gnuradio import gr +from fadingui.logger import get_logger +log = get_logger("xor_frame_sync") + class xor_frame_sync(gr.sync_block): """ @@ -26,6 +29,7 @@ class xor_frame_sync(gr.sync_block): self.nbytes = len(sync_pattern) self.nbits = len(self.pattern) + log.debug(f"Loaded pattern {self.pattern} length={self.nbits}") assert(self.nbits % 8 == 0) # packed buffer to delay the data @@ -84,12 +88,14 @@ class xor_frame_sync(gr.sync_block): self.delaybuf.appendleft(v) peak = np.argmax(self.xcorrs) - self.delay = peak - self.synchronized = True + if self.xcorrs[peak] == self.nbits: + self.delay = peak + self.synchronized = True + log.debug(f"Synchronized with delay={peak}") - if self.xcorrs[peak] != self.nbits: + else: self.synchronized = False - print(f"Warning! XOR correlation did not find a peak (max = {self.xcorrs[peak]} should be {self.nbits})") + log.warning(f"Did not find a peak (max={self.xcorrs[peak]}, should be {self.nbits})") # return data with delay -- cgit v1.2.1