From 6a57a9f39351cd57fe417506bff1fe19eaee3dba Mon Sep 17 00:00:00 2001 From: Ground0 Date: Wed, 29 Mar 2023 10:41:36 +0200 Subject: [PATCH] Removal of old caching --- inc/lcmf_main.pbi | 129 ++++++++++++++++++ inc/lweb_file_cache.pbi | 284 +++++++++++++++++++++++++++++++--------- 2 files changed, 352 insertions(+), 61 deletions(-) create mode 100644 inc/lcmf_main.pbi diff --git a/inc/lcmf_main.pbi b/inc/lcmf_main.pbi new file mode 100644 index 0000000..d6ab228 --- /dev/null +++ b/inc/lcmf_main.pbi @@ -0,0 +1,129 @@ +;******************************** +;* +;* lcmf_main.pbi +;* +;* (c) Cache (m) Memory and (f) File +;* +;* LiHaSo Webserver Modul In Memory File Cache +;* +;* It Could be configured Globaly and by Virtualhost +;* + +DeclareModule lcmf + #Cache_Server_Single = 0 + #Cache_Server_Multi = 1 + + EnumerationBinary lcmf_init + #Mem_Cache + #File_Cache + EndEnumeration + + EnumerationBinary lcmf_object_state + #NA ;Not available + #InProgress ;Copy is in Progress + #Cached ;Object is available + #Cleaning ;Object is on the way to cleaned out + EndEnumeration + + + EnumerationBinary lcmf_params_state + ;#UCP = Universal Cache Params (File and Mem) + #UCP_Max_Size_Total ;Total Size of Cache + #UCP_Max_Size_per_Entry ;Max Size of a File or Memoryblock + + ;#FAC = File Access Config + #FAC_MC ;Minimum Access Times until a File is Cached (Generic) + #FAC_RT ;Max Time a File could be in Cache (Generic) + #FAC_LA_RT ;How long a File could stay in Cache sinde last Access (Generic) + ;(If a File has regulary access For ex. every 10 min. And this lifetime is 3600min. it will be killed after FAC_RT is reached) + + #FAC_WL_Add ;Whitelist add Generic (Ex. : + ;".html" = All html files + ;"file.html" Expliciz Filenames + ;"/path/" Including all Files and Subdirectorys It is added to Valid_Source + ;"/path" only Files in Directory + ;"'/path/','.rpm'" Combines a path and a Specific file extension + + #FAC_WL_Add_Ext ;Whitelist which add with special MC/RT/LA_RT Params + #FAC_Valid_Source ;From where the files could get + #FAC_Cache_Dir ;Directory where the Cached Files could be written + + ;Some Function for Set_State who do a active Work + #FAC_Check_Cache ;Checks if all Files are the Same as Source + #UCP_Flush_Object ;Removes a Object from Cache + + EndEnumeration + + Declare.s Init(Server_Type.i = #Cache_Server_Multi) ;Create a Main Cache_Thread Management Server + Declare.s Init_cache(Cache_Type.i = #Mem_Cache) ;Create a Cache_Thread gives back the UUID + Declare.i Set_Params(Server_ID.s, Parameters.s) ;Set Cache Parameters in pre run state + Declare.i Use_New_Params(Server_ID.s) ;Reinit the Cache Thread + Declare.i Run_Cache(Server_ID.s) ;Start a initialised Cache Init()->InitCache()->Set_Params()->Run_Cache() + Declare.i Kill_Cache(Server_ID.s) ;Kills a Cache Server Thread + Declare.i Set_State(Server_ID.s, Parameters.s) ;Force to do something in Cache Thread (Flush a Element, Cleanup the Filecache-IO Intesitive, Check for Updates on Files an co.) + Declare.s Get_State(Server_ID.s, Parameters.s) ;Get Cache Server Thread State (For ex. current elements cachced, sizes and co.) + + Declare.s Put_Mem(Server_ID.s, Memblock_ToSave.i, MC.i=-1, RT.i=-1, LA_RT.i=-1) ;Push a Memory Block to cache (Ex. some pre executed scripts or pictures and so on.) + Declare.s Get_Mem(Server_ID.s, Memblock_ID.s, MC.i=-1, RT.i=-1, LA_RT.i=-1) ;Get That Memoryblock from cache who was Previously Stored + Declare.s Get_File(Server_ID.s, Full_File_Name.s, Force_Disk.i = #True) ;Get a File (Cached or uncached) could be used everytime (ServerUUID empty everytime uncached or Flag uncached is set) + +EndDeclareModule + +Module lcmf + + IncludeFile "lhs_uuid.pbi" + + Structure lcmf_mem_objects + MemoryAddress.i + State.i + EndStructure + + Structure lcmf_file_objects + LocalFileUUID.s + + EndStructure + + Structure lcmf_cache_servers + + EndStructure + + Structure lcmf_cache_servers_config_prepare + + EndStructure + + Procedure.i Main_Thread(ID.i) + ;What this Thread does: + ;* Check if Subthreads are alive and if not remove from Mapping Space + ;* Spawn new Cache Threads and Kill Cache Threads + + EndProcedure + + Procedure.i Cache_Mem_Thread(ID.i) + ;What this Thread does: + ;* Manage the mem Objects + + EndProcedure + + Procedure.i Cache_File_Thread(ID.i) + ;What this Thread does: + ;* Manage the file Objects and maybe the memobjects. + EndProcedure + + Procedure.i Copy_Thread(ID.i) + ;What this Thread does: + ;* Copy a File to Cachefolder + EndProcedure + + Procedure.i Check_Thread(ID.i) + ;What this Thread does: + ;* Checks the Cachefolder and Memorymap if all Files are valid. + EndProcedure + + Procedure.i Copy_File_to_Memory_Thread(ID.i) + ;What this thread does: + ;* Copy a local cached File to Memory + ; + EndProcedure + +EndModule + diff --git a/inc/lweb_file_cache.pbi b/inc/lweb_file_cache.pbi index 2f64d07..5aa791e 100644 --- a/inc/lweb_file_cache.pbi +++ b/inc/lweb_file_cache.pbi @@ -1,67 +1,229 @@ - +;******************************** +;* +;* lweb_file_cache.pbi +;* +;* LiHaSo Webserver Modul In Memory File Cache (Obsolete and replaced by lcmf_main.pbi) +;* +;* It Could be configured Globaly and by Virtualhost +;* + +DeclareModule lweb_FC + Enumeration Server_Type + #Type_Main ;Main Server + #Type_Virtual_Main ;Shared Virtual Server which get Data from Main + #Type_Virtual ;Standalone Virtual Server which has own Memory Area + EndEnumeration + + ;{ Error Enumeration + #Err_Unknown = -1 + #Err_Server_Not_Found = -2 + #Err_Server_File_Not_Found = -3 + ;} + + Declare.s FC_Config_Get(UUID.s) + Declare.i FC_Config_Set(UUID.s, ConfigXML.s) + Declare.i FC_Create(UUID.s, Type.i = #Type_Main) + Declare.i FC_Destroy(UUID.s) + Declare.s FC_Status_Get(UUID.s) + Declare.i FC_GetFile(UUID.s, FileName.s) + Declare.i FC_SetFile(UUID.s, FileName.s, Memory.i, Size.i) +EndDeclareModule + +Module lweb_FC + Structure St_Jobs + Type.i + Memory.i + EndStructure + + Structure St_File + lifetime.i + timer.i + size.i + memoryID.i + ThreadID.i + Accesed.i + Mutex.i + Semaphore.i + List Job.St_Jobs() + EndStructure + + Structure St_FC_Config + max_life_time.i + max_total_size.i + max_file_size.i + current_total_size.i + enabled.b + EndStructure + + Structure St_Server + VServer_UUID.s + Log_UUID.s + Status.s + Thread.i + Semaphore.i + Config.St_FC_Config + Map Files.St_File() + EndStructure + + Structure St_CacheServers + Main_UUID.s + Log_UUID.s + Status.s + Thread.i + Map Virtual.St_Server() + EndStructure + + Global Var_FC_Server.St_CacheServers + + Declare.i Cache_Cleaner(UUID.s) + Declare.i Cache_Server(UUID.s) + Declare.i Cache_File(UUID.s) + + Procedure.s FC_Config_Get(UUID.s) + Protected XMLString.s, XMLID.i + If FindMapElement(FC\Virtual(), UUID) + XMLID = CreateXML(#PB_Any) + InsertXMLMap(RootXMLNode(XMLID), FC\Virtual\ + Else + ProcedureReturn Str(#Err_Server_Not_Found) + EndIf + + EndProcedure + + Procedure.i FC_Config_Set(UUID.s, ConfigXML.s) + ;Apply the Config XML to the Map + Protected XML_ID.i + XML_ID = CreateXML(#PB_Any) + If XMLStatus(XML_ID) = #PB_XML_Success + ParseXML(XML_ID, ConfigXML) + If XMLStatus(XML_ID) = #PB_XML_Success + ExtractXMLMap(MainXMLNode(XML_ID), @FC\Virtual(UUID), #PB_XML_NoCase) + FreeXML(XML_ID) + ProcedureReturn #True + Else + FreeXML(XML_ID) + ProcedureReturn #False + EndIf + Else + If XML_ID <> 0 + FreeXML(XML_ID) + EndIf + ProcedureReturn #False + EndIf + EndProcedure + + Procedure.i FC_Create(UUID.s, Type.i = #Type_Main) + ;Create the Server + EndProcedure + + Procedure.i FC_Destroy(UUID.s) + ;Kill Thread Cache Handler Thread. + EndProcedure + + Procedure.s FC_Status_Get(UUID.s) + ;Status of the Server + EndEnumeration + + Procedure.i FC_GetFile(UUID.s, FileName.s, MemoryID.i) + ;Fill the Cached File to Memory and if response > 0 it was succesfull. + EndEnumeration + + Procedure.i FC_SetFile(UUID.s, FileName.s, Memory.i, Size.i) + ;Create cached file thread and save it to thread memory. + EndEnumeration + + Procedure.i Cache_Cleaner(UUID.s) + ;Cleans out old Files. + Protected.i Selected + Protected.i Counter + Protected.i MaxSize = Information + Repeat + Delay(1000) ;Nur Alle Sekunden Prüfen + Counter + 1 + LockMutex(file_cache_mutex) + ResetMap(m_file_cache_map()) + ;Debug "FileCacheCleaner" + Str(Counter) + " Map Size:" + Str(MapSize(m_file_cache_map())) + If NextMapElement(m_file_cache_map()) + Repeat + m_file_cache(MapKey(m_file_cache_map()))\Timer - 1 + If m_file_cache(MapKey(m_file_cache_map()))\Timer <= 0 + If m_file_cache(MapKey(m_file_cache_map()))\Buffer > 0 + configuration\cache\current - m_file_cache(MapKey(m_file_cache_map()))\Size + FreeMemory(m_file_cache(MapKey(m_file_cache_map()))\Buffer) + DeleteMapElement(m_file_cache(), MapKey(m_file_cache_map())) + lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Killed:["+MapKey(m_file_cache_map())+"]") + Selected = DeleteMapElement(m_file_cache_map()) + Else + lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Could Not kill:["+MapKey(m_file_cache_map())+"]") + EndIf + Else + Selected = NextMapElement(m_file_cache_map()) + EndIf + Until Selected = 0 + EndIf + + UnlockMutex(file_cache_mutex) + + + ForEver + EndProcedure + + Procedure.i Cache_Server(UUID.s) + ;http://purearea.net/pb/english/manual/reference/ug_memory.html + Protected FileCacheCleanerThread.i + Protected Tempbuffer.i, MaxSize.i, Current.i + MaxSize = configuration\cache\maxsize * 1024 * 1024 + FileCacheCleanerThread = CreateThread(@FileCacheCleaner(), MaxSize) + + Repeat + WaitSemaphore(file_cache_semaphore) + lhs_log_ext::Out(configuration\log\CachelogUUID, "Adresse:"+m_file_cache()\Buffer) + If (m_file_cache()\Size + configuration\cache\current) <= MaxSize + configuration\cache\current + m_file_cache()\Size + Tempbuffer = AllocateMemory(m_file_cache()\Size) + CopyMemory(m_file_cache()\Buffer, Tempbuffer, MemorySize(m_file_cache()\Buffer)) + m_file_cache()\Buffer = Tempbuffer + m_file_cache()\Timer = configuration\cache\time + m_file_cache()\Is = #True + m_file_cache_map(MapKey(m_file_cache())) = #True + SignalSemaphore(file_cache_semaphore_thread) + Else + lhs_log_ext::Out(configuration\log\CachelogUUID, "Cache Full") + SignalSemaphore(file_cache_semaphore_thread) + EndIf + + ForEver + EndProcedure + + Procedure.i Cache_File(UUID.s) + ;Procedure for File_Cache Thread who handles the Cachefile + Protected MyMemory.i, My_Host_UUID.s = StringField(UUID, 0, ";"),My_File_UUID.s = StringField(UUID, 1, ";"), WorkElement.i + Repeat + WaitSemaphore(Var_FC_Server\Virtual(UUID)\Semaphore) + LockMutex(Var_FC_Server\Virtual(UUID)\Mutex) + WorkElement = FirstElement(Var_FC_Server\Virtual(UUID)\Files() + + UnlockMutex(Var_FC_Server\Virtual(UUID)\Files(UUID)\Mutex) + + + ForEver + + + EndProcedure + + +EndModule + + ;Buggy FileCache Routines ... Wiso auch immer das noch nicht richtig Funktioniert. Procedure FileCacheCleaner(Information.i) - Protected.i Selected - Protected.i Counter - Protected.i MaxSize = Information - Repeat - Delay(1000) ;Nur Alle Sekunden Prüfen - Counter + 1 - LockMutex(file_cache_mutex) - ResetMap(m_file_cache_map()) - ;Debug "FileCacheCleaner" + Str(Counter) + " Map Size:" + Str(MapSize(m_file_cache_map())) - If NextMapElement(m_file_cache_map()) - Repeat - m_file_cache(MapKey(m_file_cache_map()))\Timer - 1 - If m_file_cache(MapKey(m_file_cache_map()))\Timer <= 0 - If m_file_cache(MapKey(m_file_cache_map()))\Buffer > 0 - configuration\cache\current - m_file_cache(MapKey(m_file_cache_map()))\Size - FreeMemory(m_file_cache(MapKey(m_file_cache_map()))\Buffer) - DeleteMapElement(m_file_cache(), MapKey(m_file_cache_map())) - lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Killed:["+MapKey(m_file_cache_map())+"]") - Selected = DeleteMapElement(m_file_cache_map()) - Else - lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Could Not kill:["+MapKey(m_file_cache_map())+"]") - EndIf - Else - Selected = NextMapElement(m_file_cache_map()) - EndIf - Until Selected = 0 - EndIf - - UnlockMutex(file_cache_mutex) - - - ForEver + EndProcedure Procedure FileCache(Information.i) - ;http://purearea.net/pb/english/manual/reference/ug_memory.html - Protected FileCacheCleanerThread.i - Protected Tempbuffer.i, MaxSize.i, Current.i - MaxSize = configuration\cache\maxsize * 1024 * 1024 - FileCacheCleanerThread = CreateThread(@FileCacheCleaner(), MaxSize) - - Repeat - WaitSemaphore(file_cache_semaphore) - lhs_log_ext::Out(configuration\log\CachelogUUID, "Adresse:"+m_file_cache()\Buffer) - If (m_file_cache()\Size + configuration\cache\current) <= MaxSize - configuration\cache\current + m_file_cache()\Size - Tempbuffer = AllocateMemory(m_file_cache()\Size) - CopyMemory(m_file_cache()\Buffer, Tempbuffer, MemorySize(m_file_cache()\Buffer)) - m_file_cache()\Buffer = Tempbuffer - m_file_cache()\Timer = configuration\cache\time - m_file_cache()\Is = #True - m_file_cache_map(MapKey(m_file_cache())) = #True - SignalSemaphore(file_cache_semaphore_thread) - Else - lhs_log_ext::Out(configuration\log\CachelogUUID, "Cache Full") - SignalSemaphore(file_cache_semaphore_thread) - EndIf - - ForEver + EndProcedure Procedure AddFileToCache(MemoryID.i, FileName.s, Size.i) @@ -92,10 +254,10 @@ Procedure.s GetFileFromCache(FileName.s, MemoryID.i) String.s = #error_string EndIf UnlockMutex(file_cache_mutex) - lhs_log_ext::Out(configuration\log\CachelogUUID, "File from Cache:"+FileName) - lhs_log_ext::Out(configuration\log\CachelogUUID, "Content: --------------------------------------------------------") - lhs_log_ext::Out(configuration\log\CachelogUUID, String) - lhs_log_ext::Out(configuration\log\CachelogUUID, "End Content. ----------------------------------------------------") + lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Info, "File from Cache:"+FileName) + lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, "Content: --------------------------------------------------------") + lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, String) + lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, "End Content. ----------------------------------------------------") ProcedureReturn String.s Else ProcedureReturn #error_string