From d3a12eaa230d57ce699b9ad6b78ee1886c9ffa72 Mon Sep 17 00:00:00 2001 From: Ground0 Date: Thu, 17 Feb 2022 13:23:42 +0100 Subject: [PATCH] Large File Support Tested with 6 GiB Files --- lweb.pbi | 137 ++++++++++++++++++++++++++++++++++++++-------- lweb.pbp | 20 +++---- lweb_header.pbi | 1 + server_example.pb | 1 + 4 files changed, 126 insertions(+), 33 deletions(-) diff --git a/lweb.pbi b/lweb.pbi index 5ce7595..5eada18 100644 --- a/lweb.pbi +++ b/lweb.pbi @@ -47,6 +47,11 @@ Module lhs_web enable.i EndStructure + Structure memory + MaxFileSize.i + DefaultBlockSize.i + EndStructure + Structure log_config Debuglog.s Debugdisable.i @@ -63,6 +68,7 @@ Module lhs_web https.server_https cache.server_cache log.log_config + mem.memory version.s identifikation.s defaultfile.s @@ -103,10 +109,14 @@ Module lhs_web configuration\https\key = "" configuration\https\key_pass = "" - configuration\cache\enable = 0 ; Enable / Disable Cached Server - configuration\cache\time = 120 ; TTL of cached files - configuration\cache\maxsize = 1 ; Max Cache - configuration\cache\current = 0 ; + configuration\cache\enable = 0 ; Enable / Disable Cached Server + configuration\cache\time = 120 ; TTL of cached files + configuration\cache\maxsize = 1 ; Max Cache + configuration\cache\current = 0 ; + + configuration\mem\MaxFileSize = 5242880 ; Default 5 MiB + configuration\mem\DefaultBlockSize = 3145728; Default Blocksize 3MiB + ;configuration\mem\DefaultBlockSize = 65536 ; Default Blocksize 64 KiB Enumeration s_client_do ;client_do_cli @@ -473,7 +483,7 @@ Module lhs_web Protected thread_cli_id = network_client_id, sent Protected MyThreadJSON, ToCall, ToCallType Protected thread_temp_cache.s, thread_temp_cache_memory, temp_receivelength, thread_temp_decode_memory - Protected thread_reasign + Protected thread_reasign, thread_oversized_file.b, thread_data_to_read.i, thread_data_readed.i Protected thread_data_size, thread_file_handle Protected.s thread_requested, thread_type, thread_date, thread_header, thread_work, JSONStringToMap, Handler_Response, response_status, PostMapString Protected thread_buffer, thread_buffer_offset, thread_buffer_length, buffer_sent @@ -634,8 +644,8 @@ Module lhs_web Handler_Response = ToCallProcedure(Header(), "") Case #handler_proto_get - Define.WebHandler_Get ToCallProcedure = ToCall - Handler_Response = ToCallProcedure(Header()) + Define.WebHandler_Get ToCallProcedureGet = ToCall + Handler_Response = ToCallProcedureGet(Header()) EndSelect lhs_log::Out("Main Client Response :"+Handler_Response) @@ -711,9 +721,21 @@ Module lhs_web EndIf thread_data_size = Lof(thread_file_handle) - thread_temp_cache_memory = AllocateMemory(thread_data_size) - ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size) - CloseFile(thread_file_handle) + ; Is File bigger than MaxFileSize in Memory allowed ? + If thread_data_size >= configuration\mem\MaxFileSize + ;Do Handle the File another way. + thread_temp_cache_memory = AllocateMemory(configuration\mem\MaxFileSize) + thread_temp_file_readed = configuration\mem\DefaultBlockSize + thread_data_readed = ReadData(thread_file_handle, thread_temp_cache_memory, thread_temp_file_readed) + thread_data_to_read = thread_data_size - thread_data_readed + thread_oversized_file = #True + Else + thread_temp_cache_memory = AllocateMemory(thread_data_size) + ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size) + CloseFile(thread_file_handle) + thread_oversized_file = #False + EndIf + ;} EndIf @@ -854,9 +876,20 @@ Module lhs_web EndIf thread_data_size = Lof(thread_file_handle) - thread_temp_cache_memory = AllocateMemory(thread_data_size) - ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size) - CloseFile(thread_file_handle) + ; Is File bigger than MaxFileSize in Memory allowed ? + If thread_data_size >= configuration\mem\MaxFileSize + ;Do Handle the File another way. + thread_temp_cache_memory = AllocateMemory(configuration\mem\MaxFileSize) + thread_temp_file_readed = configuration\mem\DefaultBlockSize + thread_data_readed = ReadData(thread_file_handle, thread_temp_cache_memory, thread_temp_file_readed) + thread_data_to_read = thread_data_size - thread_data_readed + thread_oversized_file = #True + Else + thread_temp_cache_memory = AllocateMemory(thread_data_size) + ReadData(thread_file_handle, thread_temp_cache_memory, thread_data_size) + CloseFile(thread_file_handle) + thread_oversized_file = 0 + EndIf ;} EndIf @@ -915,17 +948,29 @@ Module lhs_web Header(#http_head_connection) = "Keep-Alive" Header(#http_head_keep_alive) = "timeout=15, max=1000" thread_header = http_header_generate(Header()) - thread_buffer = AllocateMemory(thread_data_size+StringByteLength(thread_header)+12) + ;large File Handling + If thread_oversized_file = #False + thread_buffer = AllocateMemory(thread_data_size+StringByteLength(thread_header)+12) + Else + thread_buffer = AllocateMemory(thread_temp_file_readed+StringByteLength(thread_header)+12) + EndIf + thread_buffer_offset = thread_buffer thread_buffer_length = PokeS(thread_buffer_offset, thread_header,-1, #PB_UTF8|#PB_String_NoZero) : thread_buffer_offset + thread_buffer_length lhs_log::Out("Header Finished") EndIf - If thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 + ;Copy temporary File Buffer to normal Buffer. + If thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 And thread_oversized_file = #False CopyMemory(thread_temp_cache_memory, thread_buffer_offset, thread_data_size) - lhs_log::Out(Str(thread_temp_cache_memory)) + lhs_log::Out("Cache Address Memory:"+Str(thread_temp_cache_memory)) FreeMemory(thread_temp_cache_memory) - thread_temp_cache_memory=0 - Else + thread_temp_cache_memory = 0 + ElseIf thread_temp_cache_memory <> 0 And thread_buffer_offset <> 0 And thread_data_size <> 0 And thread_temp_file_readed <> 0 And thread_oversized_file = #True + CopyMemory(thread_temp_cache_memory, thread_buffer_offset, thread_temp_file_readed) + lhs_log::Out("Cache Address Memory:"+Str(thread_temp_cache_memory)) + FreeMemory(thread_temp_cache_memory) + thread_temp_cache_memory = 0 + Else lhs_log::Out("File Buffer Troubles.") If thread_temp_cache_memory = 0 : lhs_log::Out("thread_temp_cache_memory = 0") : EndIf If thread_buffer_offset = 0 : lhs_log::Out("thread_buffer_offset = 0") : EndIf @@ -944,19 +989,46 @@ Module lhs_web DeleteElement(m_clients(Str(thread_cli_id))\datenbuffer()) ;Send the data in memory to client. - sent_total = thread_data_size+(thread_buffer_offset-thread_buffer) + If thread_oversized_file = #False + sent_total = thread_data_size+(thread_buffer_offset-thread_buffer) + Else + sent_total = thread_temp_file_readed+(thread_buffer_offset-thread_buffer) + EndIf + + ;TODO: Stoped download kill server... If m_clients(Str(thread_cli_id))\client_type = #client_HTTPS sent_length = sent_total - sent_buffer_address = thread_buffer + sent_buffer_address = thread_buffer sent_total = 0 Repeat sent = ltls::WriteTLSSocket(thread_cli_id, sent_buffer_address , sent_length) If sent <> -1 - lhs_log::Out("HTTPS Sent:"+Str(sent)+" bytes") + ;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Sent:"+Str(sent)+" bytes") sent_length - sent - sent_buffer_address + sent + If thread_oversized_file = #False + sent_buffer_address + sent + Else + ;Read next Block to Memory + If sent_length <= 0 And IsFile(thread_file_handle) + sent_readed = ReadData(thread_file_handle, thread_buffer, thread_temp_file_readed) + ;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Large File -> Read Next:"+Str(sent_readed)+" bytes") + If sent_readed < thread_temp_file_readed + CloseFile(thread_file_handle) + EndIf + sent_buffer_address = thread_buffer + sent_length = sent_readed + Else + ;lhs_log::Out("ClientID:" + Str(thread_cli_id) + " HTTPS Large File -> to Send:"+Str(sent_length)+" bytes") + sent_buffer_address + sent + EndIf + EndIf + sent_total + sent Else + If IsFile(thread_file_handle) + CloseFile(thread_file_handle) + EndIf + lhs_log::Out("HTTPS Sent error:"+ltls::ErrorTLSCli(thread_cli_id)) thread_alive = #False Break 2 @@ -974,9 +1046,28 @@ Module lhs_web If sent <> -1 lhs_log::Out("HTTP Sent:"+Str(sent)+" bytes") sent_length - sent - sent_buffer_address + sent + If thread_oversized_file = 0 + sent_buffer_address + sent + Else + ;Read next Block to Memory + If sent_length <= 0 And IsFile(thread_file_handle) + sent_readed = ReadData(thread_file_handle, thread_buffer, thread_temp_file_readed) + If sent_readed < thread_temp_file_readed + CloseFile(thread_file_handle) + EndIf + sent_buffer_address = thread_buffer + sent_length = sent_readed + Else + sent_buffer_address + sent + EndIf + EndIf + sent_total + sent Else + If IsFile(thread_file_handle) + CloseFile(thread_file_handle) + EndIf + lhs_log::Out("HTTP Sent error:"+Str(sent)) thread_alive = #False Break 2 diff --git a/lweb.pbp b/lweb.pbp index 0b2bb19..2585e11 100644 --- a/lweb.pbp +++ b/lweb.pbp @@ -1,6 +1,6 @@ - +
Projekt ist inklusive Beispiel Code @@ -8,7 +8,7 @@
- +
@@ -21,7 +21,7 @@ - + @@ -61,28 +61,28 @@ - + - + - + - +
- + - + @@ -90,7 +90,7 @@ - +
diff --git a/lweb_header.pbi b/lweb_header.pbi index 0928a49..6a0d377 100644 --- a/lweb_header.pbi +++ b/lweb_header.pbi @@ -30,6 +30,7 @@ DeclareModule lhs_web #conf_error400 #conf_max_HTTP_clients #conf_max_HTTPS_clients + #conf_max_File_in_Memory #conf_server_type #conf_cache_enable #conf_HTTPS_CA diff --git a/server_example.pb b/server_example.pb index b96005e..2501528 100644 --- a/server_example.pb +++ b/server_example.pb @@ -71,6 +71,7 @@ lhs_web::set_config(lhs_web::#conf_error400, "integrated") lhs_web::set_config(lhs_web::#conf_max_HTTP_clients, "100") lhs_web::set_config(lhs_web::#conf_max_HTTPS_clients, "100") lhs_web::set_config(lhs_web::#conf_cache_enable, "0") + ;* ;* includes ;*