Large File Support Tested with 6 GiB Files

lweb.pbi

@@ -47,6 +47,11 @@ Module lhs_web
     enable.i
   EndStructure
   
+  Structure memory
+    MaxFileSize.i  
+    DefaultBlockSize.i
+  EndStructure
+  
   Structure log_config
     Debuglog.s
     Debugdisable.i
@@ -63,6 +68,7 @@ Module lhs_web
     https.server_https
     cache.server_cache
     log.log_config
+    mem.memory
     version.s
     identifikation.s
     defaultfile.s
@@ -108,6 +114,10 @@ Module lhs_web
   configuration\cache\maxsize = 1             ; Max Cache
   configuration\cache\current = 0             ;
   
+  configuration\mem\MaxFileSize = 5242880     ; Default 5 MiB
+  configuration\mem\DefaultBlockSize = 3145728; Default Blocksize 3MiB
+  ;configuration\mem\DefaultBlockSize = 65536  ; Default Blocksize 64 KiB
+  
   
   Enumeration s_client_do         ;client_do_cli
     #CLI_DO_NOP                   ;Keine Arbeit
@@ -473,7 +483,7 @@ Module lhs_web
     Protected thread_cli_id = network_client_id, sent
     Protected MyThreadJSON, ToCall, ToCallType
     Protected thread_temp_cache.s, thread_temp_cache_memory, temp_receivelength, thread_temp_decode_memory
-    Protected thread_reasign
+    Protected thread_reasign, thread_oversized_file.b, thread_data_to_read.i, thread_data_readed.i
     Protected thread_data_size, thread_file_handle
     Protected.s thread_requested, thread_type, thread_date, thread_header, thread_work, JSONStringToMap, Handler_Response, response_status, PostMapString
     Protected thread_buffer, thread_buffer_offset, thread_buffer_length, buffer_sent
@@ -634,8 +644,8 @@ Module lhs_web
                         Handler_Response = ToCallProcedure(Header(), "")
                         
                       Case #handler_proto_get
-                        Define.WebHandler_Get ToCallProcedure = ToCall
+                        Define.WebHandler_Get ToCallProcedureGet = ToCall
-                        Handler_Response = ToCallProcedure(Header())
+                        Handler_Response = ToCallProcedureGet(Header())
                         
                     EndSelect
                     lhs_log::Out("Main Client Response :"+Handler_Response)
@@ -711,9 +721,21 @@ Module lhs_web
                     EndIf
                     
                     thread_data_size = Lof(thread_file_handle)
+                    ; Is File bigger than MaxFileSize in Memory allowed ?
+                    If thread_data_size >= configuration\mem\MaxFileSize
+                      ;Do Handle the File another way.
+                      thread_temp_cache_memory = AllocateMemory(configuration\mem\MaxFileSize)
+                      thread_temp_file_readed = configuration\mem\DefaultBlockSize
+                      thread_data_readed = ReadData(thread_file_handle, thread_temp_cache_memory, thread_temp_file_readed)
+                      thread_data_to_read = thread_data_size - thread_data_readed
+                      thread_oversized_file = #True
+                    Else
                       thread_temp_cache_memory = AllocateMemory(thread_data_size)
                       ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size)
                       CloseFile(thread_file_handle)
+                      thread_oversized_file = #False
+                    EndIf
+                    
                     ;} 
                     
                   EndIf
@@ -854,9 +876,20 @@ Module lhs_web
                     EndIf
                     
                     thread_data_size = Lof(thread_file_handle)
+                    ; Is File bigger than MaxFileSize in Memory allowed ?
+                    If thread_data_size >= configuration\mem\MaxFileSize
+                      ;Do Handle the File another way.
+                      thread_temp_cache_memory = AllocateMemory(configuration\mem\MaxFileSize)
+                      thread_temp_file_readed = configuration\mem\DefaultBlockSize
+                      thread_data_readed = ReadData(thread_file_handle, thread_temp_cache_memory, thread_temp_file_readed)
+                      thread_data_to_read = thread_data_size - thread_data_readed
+                      thread_oversized_file = #True
+                    Else
                       thread_temp_cache_memory = AllocateMemory(thread_data_size)
                       ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size)
                       CloseFile(thread_file_handle)
+                      thread_oversized_file = 0
+                    EndIf
                     ;} 
                     
                   EndIf
@@ -915,16 +948,28 @@ Module lhs_web
                 Header(#http_head_connection) = "Keep-Alive"
                 Header(#http_head_keep_alive) = "timeout=15, max=1000"
                 thread_header = http_header_generate(Header())
+                ;large File Handling
+                If thread_oversized_file = #False 
                   thread_buffer = AllocateMemory(thread_data_size+StringByteLength(thread_header)+12)
+                Else
+                  thread_buffer = AllocateMemory(thread_temp_file_readed+StringByteLength(thread_header)+12)
+                EndIf
+                
                 thread_buffer_offset = thread_buffer
                 thread_buffer_length = PokeS(thread_buffer_offset, thread_header,-1, #PB_UTF8|#PB_String_NoZero) : thread_buffer_offset + thread_buffer_length
                 lhs_log::Out("Header Finished")
               EndIf
-              If thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0
+              ;Copy temporary File Buffer to normal Buffer.
+              If thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 And thread_oversized_file = #False
                 CopyMemory(thread_temp_cache_memory, thread_buffer_offset, thread_data_size)
-                lhs_log::Out(Str(thread_temp_cache_memory))
+                lhs_log::Out("Cache Address Memory:"+Str(thread_temp_cache_memory))
                 FreeMemory(thread_temp_cache_memory)
-                thread_temp_cache_memory=0
+                thread_temp_cache_memory = 0
+              ElseIf thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 And thread_temp_file_readed <> 0 And thread_oversized_file = #True
+                CopyMemory(thread_temp_cache_memory, thread_buffer_offset, thread_temp_file_readed)
+                lhs_log::Out("Cache Address Memory:"+Str(thread_temp_cache_memory))
+                FreeMemory(thread_temp_cache_memory)
+                thread_temp_cache_memory = 0
               Else             
                 lhs_log::Out("File Buffer Troubles.")
                 If thread_temp_cache_memory = 0 : lhs_log::Out("thread_temp_cache_memory = 0") : EndIf
@@ -944,7 +989,13 @@ Module lhs_web
               DeleteElement(m_clients(Str(thread_cli_id))\datenbuffer())
 
               ;Send the data in memory to client.
+              If thread_oversized_file = #False
                 sent_total = thread_data_size+(thread_buffer_offset-thread_buffer)
+              Else
+                sent_total = thread_temp_file_readed+(thread_buffer_offset-thread_buffer)
+              EndIf
+              
+              ;TODO: Stoped download kill server...
               If  m_clients(Str(thread_cli_id))\client_type = #client_HTTPS
                 sent_length = sent_total 
                 sent_buffer_address = thread_buffer                
@@ -952,11 +1003,32 @@ Module lhs_web
                 Repeat 
                   sent = ltls::WriteTLSSocket(thread_cli_id, sent_buffer_address , sent_length)
                   If sent <> -1
-                    lhs_log::Out("HTTPS Sent:"+Str(sent)+" bytes")
+                    ;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Sent:"+Str(sent)+" bytes")
                     sent_length - sent
+                    If thread_oversized_file = #False
                       sent_buffer_address + sent
+                    Else
+                      ;Read next Block to Memory
+                      If sent_length <= 0 And IsFile(thread_file_handle)
+                        sent_readed = ReadData(thread_file_handle, thread_buffer, thread_temp_file_readed)
+                        ;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Large File -> Read Next:"+Str(sent_readed)+" bytes")
+                        If sent_readed < thread_temp_file_readed
+                          CloseFile(thread_file_handle)
+                        EndIf
+                        sent_buffer_address = thread_buffer
+                        sent_length = sent_readed
+                      Else
+                        ;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Large File -> to Send:"+Str(sent_length)+" bytes")
+                        sent_buffer_address + sent
+                      EndIf                      
+                    EndIf
+                    
                     sent_total + sent
                   Else
+                    If IsFile(thread_file_handle)
+                      CloseFile(thread_file_handle)
+                    EndIf
+                    
                     lhs_log::Out("HTTPS Sent error:"+ltls::ErrorTLSCli(thread_cli_id))
                     thread_alive = #False
                     Break 2
@@ -974,9 +1046,28 @@ Module lhs_web
                   If sent <> -1
                     lhs_log::Out("HTTP Sent:"+Str(sent)+" bytes")
                     sent_length - sent
+                    If thread_oversized_file = 0
                       sent_buffer_address + sent
+                    Else
+                      ;Read next Block to Memory
+                      If sent_length <= 0 And IsFile(thread_file_handle)
+                        sent_readed = ReadData(thread_file_handle, thread_buffer, thread_temp_file_readed)
+                        If sent_readed < thread_temp_file_readed
+                          CloseFile(thread_file_handle)
+                        EndIf
+                        sent_buffer_address = thread_buffer
+                        sent_length = sent_readed
+                      Else
+                        sent_buffer_address + sent
+                      EndIf                      
+                    EndIf
+                    
                     sent_total + sent
                   Else
+                    If IsFile(thread_file_handle)
+                      CloseFile(thread_file_handle)
+                    EndIf
+                    
                     lhs_log::Out("HTTP Sent error:"+Str(sent))
                     thread_alive = #False
                     Break 2

lweb.pbp

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<project xmlns="http://www.purebasic.com/namespace" version="1.0" creator="PureBasic 5.73 LTS (Linux - x64)">
+<project xmlns="http://www.purebasic.com/namespace" version="1.0" creator="PureBasic 6.00 Beta 4 (Linux - x64)">
   <section name="config">
     <options closefiles="1" openmode="0" name="LiHaSo Webserver Modul"/>
     <comment>Projekt ist inklusive Beispiel Code</comment>
@@ -8,7 +8,7 @@
   <section name="data">
     <explorer view="../../../bin/purebasic/examples/" pattern="0"/>
     <log show="1"/>
-    <lastopen date="2022-02-14 15:46" user="renlin" host="renlin-home"/>
+    <lastopen date="2022-02-17 10:11" user="renlin" host="renlin-home"/>
   </section>
   <section name="files">
     <file name="inc/lweb_file_cache.pbi">
@@ -21,7 +21,7 @@
     </file>
     <file name="inc/lweb_helper.pbi">
       <config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+-"/>
-      <fingerprint md5="61c3c05e25271724318327b0bbd97f87"/>
+      <fingerprint md5="3a93e831dcf4bc765e6fcf05220a2553"/>
     </file>
     <file name="inc/lweb_helper_header.pbi">
       <config load="0" scan="1" panel="1" warn="1" lastopen="0" panelstate="+-"/>
@@ -61,28 +61,28 @@
     </file>
     <file name="ltls.pbi">
       <config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
-      <fingerprint md5="443cac426bd53aac4104cdc6f875b450"/>
+      <fingerprint md5="30a478c650469a17a82845db7f4007b4"/>
     </file>
     <file name="lweb.pbi">
       <config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
-      <fingerprint md5="d5f4253a4865cd5a555d702a60255df9"/>
+      <fingerprint md5="149eadfd6b7f48e17f2cadb74b429403"/>
     </file>
     <file name="lweb_header.pbi">
       <config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
-      <fingerprint md5="fa703de2feac9df61c33b9236e14eb54"/>
+      <fingerprint md5="25161e9bbf5614f5f4c1e5c072c75ed6"/>
     </file>
     <file name="server_example.pb">
       <config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
-      <fingerprint md5="542a2a146db00b759c59a819477c73b4"/>
+      <fingerprint md5="6ee2d8b7a5953dd403cd07ae8b593f67"/>
     </file>
   </section>
   <section name="targets">
     <target name="Standard-Ziel" enabled="1" default="1">
       <inputfile value="server_example.pb"/>
       <outputfile value="server_example"/>
-      <compiler version="PureBasic 5.73 LTS (Linux - x64)"/>
+      <compiler version="PureBasic 6.00 Beta 4 - C Backend (Linux - x64)"/>
       <executable value="server_example"/>
-      <options thread="1" xpskin="1" debug="1"/>
+      <options thread="1" xpskin="1" debug="1" optimizer="0"/>
       <format exe="console" cpu="0"/>
       <debugger custom="1" type="ide"/>
     </target>
@@ -90,7 +90,7 @@
       <inputfile value="lhttpd.pb"/>
       <outputfile value="lhttpd"/>
       <executable value="lhttpd"/>
-      <options thread="1"/>
+      <options thread="1" optimizer="0"/>
       <debugger custom="1" type="standalone"/>
     </target>
   </section>

lweb_header.pbi

@@ -30,6 +30,7 @@ DeclareModule lhs_web
     #conf_error400
     #conf_max_HTTP_clients
     #conf_max_HTTPS_clients
+    #conf_max_File_in_Memory
     #conf_server_type
     #conf_cache_enable
     #conf_HTTPS_CA

server_example.pb

@@ -71,6 +71,7 @@ lhs_web::set_config(lhs_web::#conf_error400, "integrated")
 lhs_web::set_config(lhs_web::#conf_max_HTTP_clients, "100")
 lhs_web::set_config(lhs_web::#conf_max_HTTPS_clients, "100")
 lhs_web::set_config(lhs_web::#conf_cache_enable, "0")
+
 ;*
 ;* includes
 ;*