From f6b1fd252e3b1b16e0e16d1f30c02dd6e19436a4 Mon Sep 17 00:00:00 2001 From: f4exb Date: Tue, 9 Oct 2018 01:06:39 +0200 Subject: [PATCH] FileRecord improvement: finalized the command to rescue corrupted .sdriq files --- .gitignore | 1 + rescuesdriq/readme.md | 59 ++++++++++++++ rescuesdriq/rescuesdriq.go | 157 ++++++++++++++++++++++++++++++------- 3 files changed, 190 insertions(+), 27 deletions(-) create mode 100644 rescuesdriq/readme.md diff --git a/.gitignore b/.gitignore index 6aa50abf1..e8a7ef5ed 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ sdrangelove.supp *.cs *.pro.user .idea/* +rescuesdriq/rescuesdriq debian/sdrangel/* debian/sdrangel.substvars debian/files diff --git a/rescuesdriq/readme.md b/rescuesdriq/readme.md new file mode 100644 index 000000000..83a33ed93 --- /dev/null +++ b/rescuesdriq/readme.md @@ -0,0 +1,59 @@ +

Repair or reformat record (.sdriq) files

+ +

Usage

+ +This utility attempts to repair .sdriq files that have their header corrupted or with a pre version 4.2.1 header. Since version 4.2.1 a CRC32 checksum is present and the file will not be played if the check of the header content against the CRC32 fails. + +The header is composed as follows: + + - Sample rate in S/s (4 bytes, 32 bits) + - Center frequency in Hz (8 bytes, 64 bits) + - Start time Unix timestamp epoch in seconds (8 bytes, 64 bits) + - Sample size as 16 or 24 bits (4 bytes, 32 bits) + - filler with all zeroes (4 bytes, 32 bits) + - CRC32 (IEEE) of the 28 bytes above (4 bytes, 32 bits) + +The header size is 32 bytes in total which is a multiple of 8 bytes thus occupies an integer number of samples whether in 16 or 24 bits mode. When migrating from a pre version 4.2.1 header you may crunch a very small amount of samples. + +You can replace values in the header with the following options: + + - -sr uint + Sample rate (S/s) + - -cf uint + Center frequency (Hz) + - -ts string + start time RFC3339 (ex: 2006-01-02T15:04:05Z) + - -now + use now for start time + - -sz uint + Sample size (16 or 24) (default 16) + +You need to specify an input file. If no output file is specified the current header values are printed to the console and the program exits: + + - -in string + input file (default "foo") + +To convert to a new file you need to specify the output file: + + - -out string + output file (default "foo") + +You can specify a block size in multiples of 4k for the copy. Large blocks will yield a faster copy but a larger output file. With the default of 1 (4k) the copy does not take much time anyway: + + - -bz uint + Copy block size in multiple of 4k (default 1) + +

Build

+ +The program is written in go and is provided only in source code form. Compiling it is very easy: + +

Install go

+ +You will usually find a `golang` package in your distribution. For example in Ubuntu or Debian you can install it with `sudo apt-get install golang`. You can find binary distributions for many systems at the [Go download site](https://golang.org/dl/) + +

Build the program

+ +In this directory just do `go build` + + + \ No newline at end of file diff --git a/rescuesdriq/rescuesdriq.go b/rescuesdriq/rescuesdriq.go index 614fa4216..fdf4108ab 100644 --- a/rescuesdriq/rescuesdriq.go +++ b/rescuesdriq/rescuesdriq.go @@ -9,6 +9,7 @@ import ( "bytes" "encoding/binary" "time" + "hash/crc32" ) @@ -17,27 +18,18 @@ type HeaderStd struct { CenterFrequency uint64 StartTimestamp int64 SampleSize uint32 - _ uint32 + Filler uint32 CRC32 uint32 } -func analyze(fileName string) HeaderStd { - fmt.Println("input file:", fileName) - // open input file - fi, err := os.Open(fileName) - if err != nil { - panic(err) +func check(e error) { + if e != nil { + panic(e) } - // close fi on exit and check for its returned error - defer func() { - if err := fi.Close(); err != nil { - panic(err) - } - }() - // make a read buffer - r := bufio.NewReader(fi) - +} + +func analyze(r *bufio.Reader) HeaderStd { headerbuf := make([]byte, 32) // This is a full header with CRC n, err := r.Read(headerbuf) if err != nil && err != io.EOF { @@ -50,27 +42,138 @@ func analyze(fileName string) HeaderStd { var header HeaderStd headerr := bytes.NewReader(headerbuf) err = binary.Read(headerr, binary.LittleEndian, &header) - if err != nil { - panic(err) - } - - fmt.Println("Sample rate:", header.SampleRate) - fmt.Println("Frequency :", header.CenterFrequency) - fmt.Println("Sample Size:", header.SampleSize) - tm := time.Unix(header.StartTimestamp, 0) - fmt.Println("Start :", tm) + check(err) return header } +func writeHeader(writer *bufio.Writer, header *HeaderStd) { + var bin_buf bytes.Buffer + binary.Write(&bin_buf, binary.LittleEndian, header) + noh, err := writer.Write(bin_buf.Bytes()) + check(err) + fmt.Printf("Wrote %d bytes header\n", noh) +} + +func printHeader(header *HeaderStd) { + fmt.Println("Sample rate:", header.SampleRate) + fmt.Println("Frequency :", header.CenterFrequency) + fmt.Println("Sample Size:", header.SampleSize) + tm := time.Unix(header.StartTimestamp, 0) + fmt.Println("Start :", tm) +} + +func setCRC(header *HeaderStd) { + var bin_buf bytes.Buffer + header.Filler = 0 + binary.Write(&bin_buf, binary.LittleEndian, header) + header.CRC32 = crc32.ChecksumIEEE(bin_buf.Bytes()[0:28]) +} + +func copyContent(reader *bufio.Reader, writer *bufio.Writer, blockSize uint) { + p := make([]byte, blockSize*4096) // buffer in 4k multiples + var sz int64 = 0 + + for { + n, err := reader.Read(p) + + if err != nil { + if err == io.EOF { + writer.Write(p[0:n]) + sz += int64(n) + break + } else { + fmt.Println("An error occured during content copy. Aborting") + break + } + } else { + writer.Write(p) + sz += int64(blockSize)*4096 + } + + fmt.Printf("Wrote %d bytes\r", sz) + } + + fmt.Printf("Wrote %d bytes\r", sz) +} + func main() { - wordPtr := flag.String("in", "foo", "input file") + inFileStr := flag.String("in", "foo", "input file") + outFileStr := flag.String("out", "foo", "output file") + sampleRate := flag.Uint("sr", 0, "Sample rate (S/s)") + centerFreq := flag.Uint64("cf", 0, "Center frequency (Hz)") + sampleSize := flag.Uint("sz", 16, "Sample size (16 or 24)") + timeStr := flag.String("ts", "", "start time RFC3339 (ex: 2006-01-02T15:04:05Z)") + timeNow := flag.Bool("now", false , "use now for start time") + blockSize := flag.Uint("bz", 1, "Copy block size in multiple of 4k") + flag.Parse() flagSeen := make(map[string]bool) flag.Visit(func(f *flag.Flag) { flagSeen[f.Name] = true }) if flagSeen["in"] { - analyze(*wordPtr) + fmt.Println("input file :", *inFileStr) + + // open input file + fi, err := os.Open(*inFileStr) + check(err) + // close fi on exit and check for its returned error + defer func() { + err := fi.Close(); + check(err) + }() + // make a read buffer + reader := bufio.NewReader(fi) + var headerOrigin HeaderStd = analyze(reader) + printHeader(&headerOrigin) + + if flagSeen["out"] { + if flagSeen["sr"] { + headerOrigin.SampleRate = uint32(*sampleRate) + } else if flagSeen["cf"] { + headerOrigin.CenterFrequency = *centerFreq + } else if flagSeen["sz"] { + if (*sampleSize == 16) || (*sampleSize == 24) { + headerOrigin.SampleSize = uint32(*sampleSize) + } else { + fmt.Println("Incorrect sample size specified. Defaulting to 16") + headerOrigin.SampleSize = 16 + } + } else if flagSeen["ts"] { + t, err := time.Parse(time.RFC3339, *timeStr) + if (err == nil) { + headerOrigin.StartTimestamp = t.Unix() + } else { + fmt.Println("Incorrect time specified. Defaulting to now") + headerOrigin.StartTimestamp = int64(time.Now().Unix()) + } + } else if *timeNow { + headerOrigin.StartTimestamp = int64(time.Now().Unix()) + } + + fmt.Println("\nHeader is now") + printHeader(&headerOrigin) + setCRC(&headerOrigin) + fmt.Println("CRC32 :", headerOrigin.CRC32) + fmt.Println("Output file:", *outFileStr) + + fo, err := os.Create(*outFileStr) + check(err) + + defer func() { + err := fo.Close(); + check(err) + }() + + writer := bufio.NewWriter(fo) + + writeHeader(writer, &headerOrigin) + copyContent(reader, writer, *blockSize) + + fmt.Println("\nCopy done") + writer.Flush() + } + } else { fmt.Println("No input file given") }