PB_Includes/lweb
1
0
Fork 0
Code Issues 3 Pull requests Projects 1 Releases 2 Wiki Activity

Large File Support Tested with 6 GiB Files

Browse source
This commit is contained in:
René Linder 2022-02-17 13:23:42 +01:00
parent 421dc9b04f
commit d3a12eaa23
4 changed files with 126 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

137
lweb.pbi
View file

@ -47,6 +47,11 @@ Module lhs_web
enable.i
EndStructure
Structure memory
MaxFileSize.i
DefaultBlockSize.i
EndStructure
Structure log_config
Debuglog.s
Debugdisable.i
@ -63,6 +68,7 @@ Module lhs_web
https.server_https
cache.server_cache
log.log_config
mem.memory
version.s
identifikation.s
defaultfile.s
@ -103,10 +109,14 @@ Module lhs_web
configuration\https\key = ""
configuration\https\key_pass = ""
configuration\cache\enable = 0 ; Enable / Disable Cached Server
configuration\cache\time = 120 ; TTL of cached files
configuration\cache\maxsize = 1 ; Max Cache
configuration\cache\current = 0 ;
configuration\cache\enable = 0 ; Enable / Disable Cached Server
configuration\cache\time = 120 ; TTL of cached files
configuration\cache\maxsize = 1 ; Max Cache
configuration\cache\current = 0 ;
configuration\mem\MaxFileSize = 5242880 ; Default 5 MiB
configuration\mem\DefaultBlockSize = 3145728; Default Blocksize 3MiB
;configuration\mem\DefaultBlockSize = 65536 ; Default Blocksize 64 KiB
Enumeration s_client_do ;client_do_cli
@ -473,7 +483,7 @@ Module lhs_web
Protected thread_cli_id = network_client_id, sent
Protected MyThreadJSON, ToCall, ToCallType
Protected thread_temp_cache.s, thread_temp_cache_memory, temp_receivelength, thread_temp_decode_memory
Protected thread_reasign
Protected thread_reasign, thread_oversized_file.b, thread_data_to_read.i, thread_data_readed.i
Protected thread_data_size, thread_file_handle
Protected.s thread_requested, thread_type, thread_date, thread_header, thread_work, JSONStringToMap, Handler_Response, response_status, PostMapString
Protected thread_buffer, thread_buffer_offset, thread_buffer_length, buffer_sent
@ -634,8 +644,8 @@ Module lhs_web
Handler_Response = ToCallProcedure(Header(), "")
Case #handler_proto_get
Define.WebHandler_Get ToCallProcedure = ToCall
Handler_Response = ToCallProcedure(Header())
Define.WebHandler_Get ToCallProcedureGet = ToCall
Handler_Response = ToCallProcedureGet(Header())
EndSelect
lhs_log::Out("Main Client Response :"+Handler_Response)
@ -711,9 +721,21 @@ Module lhs_web
EndIf
thread_data_size = Lof(thread_file_handle)
thread_temp_cache_memory = AllocateMemory(thread_data_size)
ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size)
CloseFile(thread_file_handle)
; Is File bigger than MaxFileSize in Memory allowed ?
If thread_data_size >= configuration\mem\MaxFileSize
;Do Handle the File another way.
thread_temp_cache_memory = AllocateMemory(configuration\mem\MaxFileSize)
thread_temp_file_readed = configuration\mem\DefaultBlockSize
thread_data_readed = ReadData(thread_file_handle, thread_temp_cache_memory, thread_temp_file_readed)
thread_data_to_read = thread_data_size - thread_data_readed
thread_oversized_file = #True
Else
thread_temp_cache_memory = AllocateMemory(thread_data_size)
ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size)
CloseFile(thread_file_handle)
thread_oversized_file = #False
EndIf
;}
EndIf
@ -854,9 +876,20 @@ Module lhs_web
EndIf
thread_data_size = Lof(thread_file_handle)
thread_temp_cache_memory = AllocateMemory(thread_data_size)
ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size)
CloseFile(thread_file_handle)
; Is File bigger than MaxFileSize in Memory allowed ?
If thread_data_size >= configuration\mem\MaxFileSize
;Do Handle the File another way.
thread_temp_cache_memory = AllocateMemory(configuration\mem\MaxFileSize)
thread_temp_file_readed = configuration\mem\DefaultBlockSize
thread_data_readed = ReadData(thread_file_handle, thread_temp_cache_memory, thread_temp_file_readed)
thread_data_to_read = thread_data_size - thread_data_readed
thread_oversized_file = #True
Else
thread_temp_cache_memory = AllocateMemory(thread_data_size)
ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size)
CloseFile(thread_file_handle)
thread_oversized_file = 0
EndIf
;}
EndIf
@ -915,17 +948,29 @@ Module lhs_web
Header(#http_head_connection) = "Keep-Alive"
Header(#http_head_keep_alive) = "timeout=15, max=1000"
thread_header = http_header_generate(Header())
thread_buffer = AllocateMemory(thread_data_size+StringByteLength(thread_header)+12)
;large File Handling
If thread_oversized_file = #False
thread_buffer = AllocateMemory(thread_data_size+StringByteLength(thread_header)+12)
Else
thread_buffer = AllocateMemory(thread_temp_file_readed+StringByteLength(thread_header)+12)
EndIf
thread_buffer_offset = thread_buffer
thread_buffer_length = PokeS(thread_buffer_offset, thread_header,-1, #PB_UTF8|#PB_String_NoZero) : thread_buffer_offset + thread_buffer_length
lhs_log::Out("Header Finished")
EndIf
If thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0
;Copy temporary File Buffer to normal Buffer.
If thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 And thread_oversized_file = #False
CopyMemory(thread_temp_cache_memory, thread_buffer_offset, thread_data_size)
lhs_log::Out(Str(thread_temp_cache_memory))
lhs_log::Out("Cache Address Memory:"+Str(thread_temp_cache_memory))
FreeMemory(thread_temp_cache_memory)
thread_temp_cache_memory=0
Else
thread_temp_cache_memory = 0
ElseIf thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 And thread_temp_file_readed <> 0 And thread_oversized_file = #True
CopyMemory(thread_temp_cache_memory, thread_buffer_offset, thread_temp_file_readed)
lhs_log::Out("Cache Address Memory:"+Str(thread_temp_cache_memory))
FreeMemory(thread_temp_cache_memory)
thread_temp_cache_memory = 0
Else
lhs_log::Out("File Buffer Troubles.")
If thread_temp_cache_memory = 0 : lhs_log::Out("thread_temp_cache_memory = 0") : EndIf
If thread_buffer_offset = 0 : lhs_log::Out("thread_buffer_offset = 0") : EndIf
@ -944,19 +989,46 @@ Module lhs_web
DeleteElement(m_clients(Str(thread_cli_id))\datenbuffer())
;Send the data in memory to client.
sent_total = thread_data_size+(thread_buffer_offset-thread_buffer)
If thread_oversized_file = #False
sent_total = thread_data_size+(thread_buffer_offset-thread_buffer)
Else
sent_total = thread_temp_file_readed+(thread_buffer_offset-thread_buffer)
EndIf
;TODO: Stoped download kill server...
If m_clients(Str(thread_cli_id))\client_type = #client_HTTPS
sent_length = sent_total
sent_buffer_address = thread_buffer
sent_buffer_address = thread_buffer
sent_total = 0
Repeat
sent = ltls::WriteTLSSocket(thread_cli_id, sent_buffer_address , sent_length)
If sent <> -1
lhs_log::Out("HTTPS Sent:"+Str(sent)+" bytes")
;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Sent:"+Str(sent)+" bytes")
sent_length - sent
sent_buffer_address + sent
If thread_oversized_file = #False
sent_buffer_address + sent
Else
;Read next Block to Memory
If sent_length <= 0 And IsFile(thread_file_handle)
sent_readed = ReadData(thread_file_handle, thread_buffer, thread_temp_file_readed)
;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Large File -> Read Next:"+Str(sent_readed)+" bytes")
If sent_readed < thread_temp_file_readed
CloseFile(thread_file_handle)
EndIf
sent_buffer_address = thread_buffer
sent_length = sent_readed
Else
;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Large File -> to Send:"+Str(sent_length)+" bytes")
sent_buffer_address + sent
EndIf
EndIf
sent_total + sent
Else
If IsFile(thread_file_handle)
CloseFile(thread_file_handle)
EndIf
lhs_log::Out("HTTPS Sent error:"+ltls::ErrorTLSCli(thread_cli_id))
thread_alive = #False
Break 2
@ -974,9 +1046,28 @@ Module lhs_web
If sent <> -1
lhs_log::Out("HTTP Sent:"+Str(sent)+" bytes")
sent_length - sent
sent_buffer_address + sent
If thread_oversized_file = 0
sent_buffer_address + sent
Else
;Read next Block to Memory
If sent_length <= 0 And IsFile(thread_file_handle)
sent_readed = ReadData(thread_file_handle, thread_buffer, thread_temp_file_readed)
If sent_readed < thread_temp_file_readed
CloseFile(thread_file_handle)
EndIf
sent_buffer_address = thread_buffer
sent_length = sent_readed
Else
sent_buffer_address + sent
EndIf
EndIf
sent_total + sent
Else
If IsFile(thread_file_handle)
CloseFile(thread_file_handle)
EndIf
lhs_log::Out("HTTP Sent error:"+Str(sent))
thread_alive = #False
Break 2

20
lweb.pbp
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://www.purebasic.com/namespace" version="1.0" creator="PureBasic 5.73 LTS (Linux - x64)">
<project xmlns="http://www.purebasic.com/namespace" version="1.0" creator="PureBasic 6.00 Beta 4 (Linux - x64)">
<section name="config">
<options closefiles="1" openmode="0" name="LiHaSo Webserver Modul"/>
<comment>Projekt ist inklusive Beispiel Code</comment>
@ -8,7 +8,7 @@
<section name="data">
<explorer view="../../../bin/purebasic/examples/" pattern="0"/>
<log show="1"/>
<lastopen date="2022-02-14 15:46" user="renlin" host="renlin-home"/>
<lastopen date="2022-02-17 10:11" user="renlin" host="renlin-home"/>
</section>
<section name="files">
<file name="inc/lweb_file_cache.pbi">
@ -21,7 +21,7 @@
</file>
<file name="inc/lweb_helper.pbi">
<config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+-"/>
<fingerprint md5="61c3c05e25271724318327b0bbd97f87"/>
<fingerprint md5="3a93e831dcf4bc765e6fcf05220a2553"/>
</file>
<file name="inc/lweb_helper_header.pbi">
<config load="0" scan="1" panel="1" warn="1" lastopen="0" panelstate="+-"/>
@ -61,28 +61,28 @@
</file>
<file name="ltls.pbi">
<config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
<fingerprint md5="443cac426bd53aac4104cdc6f875b450"/>
<fingerprint md5="30a478c650469a17a82845db7f4007b4"/>
</file>
<file name="lweb.pbi">
<config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
<fingerprint md5="d5f4253a4865cd5a555d702a60255df9"/>
<fingerprint md5="149eadfd6b7f48e17f2cadb74b429403"/>
</file>
<file name="lweb_header.pbi">
<config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
<fingerprint md5="fa703de2feac9df61c33b9236e14eb54"/>
<fingerprint md5="25161e9bbf5614f5f4c1e5c072c75ed6"/>
</file>
<file name="server_example.pb">
<config load="0" scan="1" panel="1" warn="1" lastopen="1" panelstate="+"/>
<fingerprint md5="542a2a146db00b759c59a819477c73b4"/>
<fingerprint md5="6ee2d8b7a5953dd403cd07ae8b593f67"/>
</file>
</section>
<section name="targets">
<target name="Standard-Ziel" enabled="1" default="1">
<inputfile value="server_example.pb"/>
<outputfile value="server_example"/>
<compiler version="PureBasic 5.73 LTS (Linux - x64)"/>
<compiler version="PureBasic 6.00 Beta 4 - C Backend (Linux - x64)"/>
<executable value="server_example"/>
<options thread="1" xpskin="1" debug="1"/>
<options thread="1" xpskin="1" debug="1" optimizer="0"/>
<format exe="console" cpu="0"/>
<debugger custom="1" type="ide"/>
</target>
@ -90,7 +90,7 @@
<inputfile value="lhttpd.pb"/>
<outputfile value="lhttpd"/>
<executable value="lhttpd"/>
<options thread="1"/>
<options thread="1" optimizer="0"/>
<debugger custom="1" type="standalone"/>
</target>
</section>

1
lweb_header.pbi
View file

@ -30,6 +30,7 @@ DeclareModule lhs_web
#conf_error400
#conf_max_HTTP_clients
#conf_max_HTTPS_clients
#conf_max_File_in_Memory
#conf_server_type
#conf_cache_enable
#conf_HTTPS_CA

1
server_example.pb
View file

@ -71,6 +71,7 @@ lhs_web::set_config(lhs_web::#conf_error400, "integrated")
lhs_web::set_config(lhs_web::#conf_max_HTTP_clients, "100")
lhs_web::set_config(lhs_web::#conf_max_HTTPS_clients, "100")
lhs_web::set_config(lhs_web::#conf_cache_enable, "0")
;*
;* includes
;*