Cautionary tale of open, append, close

[mdijkens]

Good point!
Beside keeping files open, writing millions of small strings is also expensive.
I've adapted my writeBigFile function for your scenario to append to multiple outputfiles (~250 MB/sec)
I'm using this approach to merge logfiles of over 100GB each

Code: (Select All)

'open all output files

Dim fh%(0 To 9)

For i% = 0 To 9

  fh%(i%) = writeBigFile(0, "~test" + _ToStr$(i%) + ".tmp")

Next i%



'append 10mln random string to random output files

t0! = Timer

For n& = 1 To 10000000

  i% = Int(Rnd * 10)

  v$ = String$(1 + Rnd * 100, 33 + Rnd * 30)

  tbytes&& = tbytes&& + writeBigFile(fh%(i%), v$)

Next n&

t1! = Timer



'close all output files

For i% = 0 To 9

  Print Using "~test#.tmp ###,###,### bytes"; i%; writeBigFile(-fh%(i%), "")

Next i%

Print Using "###,###,### total bytes written in #.### seconds"; tbytes&&; t1! - t0!



Function writeBigFile~&& (file%, content$)

  Const BLOCKSIZE = 2 ^ 22 ' 4 MB

  Static buf$(1000), bufPos(1000) As _Integer64

  Dim As _Unsigned _Integer64 contentLen, bufRemain



  If file% > 0 Then 'append file

    contentLen = Len(content$): bufRemain = BLOCKSIZE - bufPos(file%)

    writeBigFile = contentLen

    If contentLen > bufRemain Then

      Mid$(buf$(file%), bufPos(file%) + 1, bufRemain) = Left$(content$, bufRemain)

      Print #file%, buf$(file%);

      content$ = Mid$(content$, bufRemain + 1)

      contentLen = contentLen - bufRemain

      bufPos(file%) = 0

    End If

    Mid$(buf$(file%), bufPos(file%) + 1, contentLen) = Left$(content$, contentLen)

    bufPos(file%) = bufPos(file%) + contentLen

  ElseIf file% = 0 Then 'new file

    file% = FreeFile

    buf$(file%) = String$(BLOCKSIZE, 0): bufPos(file%) = 0

    Open content$ For Append As #file%

    writeBigFile = file%

  ElseIf file% < 0 Then 'close file

    file% = -file%

    If bufPos(file%) > 0 Then Print #file%, Left$(buf$(file%), bufPos(file%));

    buf$(file%) = ""

    writeBigFile = LOF(file%)

    Close #file%

  End If

End Function

You can change APPEND to OUTPUT in line 42 if you want to overwrite output files