(10-21-2023, 01:36 PM)mdijkens Wrote: This is a simplified part of a more complex process to split 1 huge inFile into multiple smaller outFile() ones:

Code: (Select All)

  Dim As _Unsigned _Byte inPos, splitFiles, splitFile
  Dim As _Unsigned _Integer64 inSize, splitPos, splitSize
  Dim As _Unsigned _Byte inFile(1 To inSize), char
  Get #1, 1, inFile()
  Dim As String outFile(splitFiles)
  For splitFile = 1 To splitFiles
    outFile(splitFile) = String$(splitSize, 0)
  Next splitFile

  For splitPos = 1 To splitSize
    For splitFile = 1 To splitFiles
      inPos = inPos + 1
      If inPos <= inSize Then
        char = inFile(inPos)
        Mid$(outFile(splitFile), splitPos, 1) = Chr$(char)
      End If
    Next splitFile
  Next splitPos
  For splitFile = 1 To splitFiles
    Put #splitFile%, , outFile(splitFile)
  Next splitFile

inFile is the byte-array of the inputfile
inSize is the size in bytes of the inFile
inPos is the current characterposition of the inFile
outFile() are the strings build for the split-files
splitFiles is the number of files to split into
splitSize is the size in bytes of each outFile (e.g. roundup(inSize/splitFiles))
splitFile is current splitFile
splitPos is the current characterposition of the outFile

Above works, but variable length strings and the Mid$() command are very time-expensive (2GB inFile takes >10 minutes)

I've tried 2-dimensional byte-arrays for the out-files like outFile(files, length) , but QB64 does not support Put with one dimension like Put #x, , outFile(x)
I've also tried mapping this 2-dimensional array with _MEM but did not succeed so far.

Does anyone have a clever trick to speed this up?

_DEFINE A-Z AS _INTEGER64

PRINT "Creating file data"

'create a nice large temp file

a$ = "Hello World.  I'm Testing Crap!"

FOR i = 1 TO 26

    a$ = a$ + a$

NEXT





OPEN "test.txt" FOR BINARY AS #1

PRINT "Writing file data", LEN(a$)

PUT #1, , a$

CLOSE



infile$ = "test.txt" 'the file to read in and split

savefile$ = "split test" 'the name of the files to save to, without extension or index

Splitfiles = 11 'split it into 11 parts (or whatever you want)





OPEN infile$ FOR BINARY AS #1

SplitFileSize = _CEIL(LOF(1) / Splitfiles)

PRINT "Splitting file data", SplitFileSize







t## = TIMER

i = 0: p = 1

DO UNTIL EOF(1) OR p > LOF(1)

    i = i + 1

    IF p + SplitFileSize > LOF(1) THEN SplitFileSize = LOF(1) - SplitFileSize * (i - 1)

    partialfile$ = SPACE$(SplitFileSize)

    GET #1, , partialfile$

    p = p + SplitFileSize

    outfile$ = savefile$ + "(" + _TRIM$(STR$(i)) + ").split"

    PRINT "Writing file: "; outfile$, LEN(partialfile$)

    OPEN outfile$ FOR OUTPUT AS #2: CLOSE #2

    OPEN outfile$ FOR BINARY AS #2

    PUT #2, , partialfile$

    CLOSE #2

    IF i > 100 THEN Explode

LOOP

PRINT USING "Done in ###.### seconds."; TIMER - t##



SUB Explode

    BEEP

    BEEP

    BEEP

    PRINT "OMG!! WE just filled the drive with a ton of crap from a bad, endless split!!"

    BEEP

    BEEP

    BEEP

    PRINT "HELP!!"

    BEEP

    BEEP

    BEEP

    PRINT "HELP!!"

    BEEP

    BEEP

    BEEP

    PRINT "DELETE IT ALL!!  HEEEEEELLLPPPP!!!"

    BEEP

    BEEP

    BEEP

    END

END SUB

'$CHECKING:OFF

_DEFINE A-Z AS _INTEGER64

PRINT "Creating file data"

'create a nice large temp file

a$ = "0123456789"

FOR i = 1 TO 27

    a$ = a$ + a$

NEXT



OPEN "test.txt" FOR OUTPUT AS #1: CLOSE #1

OPEN "test.txt" FOR BINARY AS #1

PRINT "Writing file data", LEN(a$) '1.3 GB file

PUT #1, , a$

CLOSE



DIM FileData AS _MEM

FileData = _MEMNEW(LEN(a$)) 'can't point a mem pointer to a variable length string, so make a new memblock

_MEMPUT FileData, FileData.OFFSET, a$ 'and put the string in it



DIM splitData(0 TO 9) AS STRING





OPEN "test.txt" FOR BINARY AS #1



FOR i = 0 TO 9

    OPEN "test_split(" + _TRIM$(STR$(i)) + ").txt" FOR OUTPUT AS #i + 2: CLOSE #i + 2

    OPEN "test_split(" + _TRIM$(STR$(i)) + ").txt" FOR BINARY AS #i + 2

    splitData(i) = SPACE$(1000)

NEXT



PRINT "Writing Data..."

t## = TIMER



DIM o AS _OFFSET

o = FileData.OFFSET



c = 1 'c is the push counter



DO UNTIL o >= FileData.OFFSET + FileData.SIZE

    FOR i = 0 TO 9

        ASC(splitData(i), c) = _MEMGET(FileData, o + i, _UNSIGNED _BYTE)

    NEXT

    IF c MOD 1000 = 0 THEN 'dump data in small segments so our strings don't grow too large

        c = 0

        FOR j = 0 TO 9

            PUT 2 + j, , splitData(j)

        NEXT

    END IF

    o = o + 10

    c = c + 1

LOOP

IF c <> 1 THEN 'dump remained of data

    FOR j = 0 TO 9

        l$ = LEFT$(splitData(j), c - 1)

        PUT 2 + j, , l$

        splitData(j) = ""

    NEXT

END IF



PRINT USING "###.### seconds to break data into 10 files."; TIMER - t##